2024-11-24 11:14:10,587 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 11:14:10,603 main DEBUG Took 0.013201 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 11:14:10,603 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 11:14:10,603 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 11:14:10,605 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 11:14:10,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,615 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 11:14:10,627 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,629 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,630 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,631 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,632 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,633 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,634 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,634 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,635 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,635 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,636 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,637 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,637 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,637 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 11:14:10,638 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,638 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 11:14:10,640 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 11:14:10,641 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 11:14:10,643 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 11:14:10,643 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 11:14:10,644 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 11:14:10,645 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 11:14:10,652 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 11:14:10,654 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 11:14:10,655 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 11:14:10,656 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 11:14:10,656 main DEBUG createAppenders(={Console}) 2024-11-24 11:14:10,657 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-24 11:14:10,657 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 11:14:10,657 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-24 11:14:10,658 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 11:14:10,658 main DEBUG OutputStream closed 2024-11-24 11:14:10,658 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 11:14:10,659 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 11:14:10,659 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-24 11:14:10,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 11:14:10,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 11:14:10,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 11:14:10,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 11:14:10,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 11:14:10,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 11:14:10,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 11:14:10,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 11:14:10,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 11:14:10,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 11:14:10,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 11:14:10,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 11:14:10,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 11:14:10,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 11:14:10,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 11:14:10,740 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 11:14:10,740 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 11:14:10,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 11:14:10,743 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 11:14:10,743 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-24 11:14:10,744 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 11:14:10,744 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-24T11:14:10,983 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca 2024-11-24 11:14:10,988 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 11:14:10,988 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T11:14:10,997 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-24T11:14:11,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T11:14:11,039 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161, deleteOnExit=true 2024-11-24T11:14:11,040 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T11:14:11,041 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/test.cache.data in system properties and HBase conf 2024-11-24T11:14:11,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T11:14:11,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/hadoop.log.dir in system properties and HBase conf 2024-11-24T11:14:11,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T11:14:11,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T11:14:11,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T11:14:11,148 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T11:14:11,236 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T11:14:11,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T11:14:11,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T11:14:11,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T11:14:11,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T11:14:11,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T11:14:11,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T11:14:11,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T11:14:11,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T11:14:11,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T11:14:11,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/nfs.dump.dir in system properties and HBase conf 2024-11-24T11:14:11,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/java.io.tmpdir in system properties and HBase conf 2024-11-24T11:14:11,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T11:14:11,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T11:14:11,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T11:14:12,326 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T11:14:12,394 INFO [Time-limited test {}] log.Log(170): Logging initialized @2502ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T11:14:12,457 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T11:14:12,512 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T11:14:12,534 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T11:14:12,534 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T11:14:12,536 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T11:14:12,547 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T11:14:12,550 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/hadoop.log.dir/,AVAILABLE} 2024-11-24T11:14:12,551 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T11:14:12,725 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/java.io.tmpdir/jetty-localhost-38779-hadoop-hdfs-3_4_1-tests_jar-_-any-14599646425376674963/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T11:14:12,732 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:38779} 2024-11-24T11:14:12,732 INFO [Time-limited test {}] server.Server(415): Started @2840ms 2024-11-24T11:14:13,274 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T11:14:13,282 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T11:14:13,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T11:14:13,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T11:14:13,283 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T11:14:13,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/hadoop.log.dir/,AVAILABLE} 2024-11-24T11:14:13,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T11:14:13,384 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/java.io.tmpdir/jetty-localhost-35905-hadoop-hdfs-3_4_1-tests_jar-_-any-8685097650593436548/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T11:14:13,384 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:35905} 2024-11-24T11:14:13,385 INFO [Time-limited test {}] server.Server(415): Started @3493ms 2024-11-24T11:14:13,432 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T11:14:14,396 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/data/data1/current/BP-1825513930-172.17.0.3-1732446851790/current, will proceed with Du for space computation calculation, 2024-11-24T11:14:14,396 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/data/data2/current/BP-1825513930-172.17.0.3-1732446851790/current, will proceed with Du for space computation calculation, 2024-11-24T11:14:14,458 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T11:14:14,501 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaa853e2bb37c7011 with lease ID 0x89d04e57227c5b5c: Processing first storage report for DS-6350b056-46ca-4ffe-9891-f23b1a6da2c1 from datanode DatanodeRegistration(127.0.0.1:34189, datanodeUuid=2f4fbaac-9b8c-4e23-b847-b1bcf7187605, infoPort=33579, infoSecurePort=0, ipcPort=46647, storageInfo=lv=-57;cid=testClusterID;nsid=1749253065;c=1732446851790) 2024-11-24T11:14:14,502 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaa853e2bb37c7011 with lease ID 0x89d04e57227c5b5c: from storage DS-6350b056-46ca-4ffe-9891-f23b1a6da2c1 node DatanodeRegistration(127.0.0.1:34189, datanodeUuid=2f4fbaac-9b8c-4e23-b847-b1bcf7187605, infoPort=33579, infoSecurePort=0, ipcPort=46647, storageInfo=lv=-57;cid=testClusterID;nsid=1749253065;c=1732446851790), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T11:14:14,503 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaa853e2bb37c7011 with lease ID 0x89d04e57227c5b5c: Processing first storage report for DS-5a597e02-88d4-46d0-ae27-af92d125fe4b from datanode DatanodeRegistration(127.0.0.1:34189, datanodeUuid=2f4fbaac-9b8c-4e23-b847-b1bcf7187605, infoPort=33579, infoSecurePort=0, ipcPort=46647, storageInfo=lv=-57;cid=testClusterID;nsid=1749253065;c=1732446851790) 2024-11-24T11:14:14,503 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaa853e2bb37c7011 with lease ID 0x89d04e57227c5b5c: from storage DS-5a597e02-88d4-46d0-ae27-af92d125fe4b node DatanodeRegistration(127.0.0.1:34189, datanodeUuid=2f4fbaac-9b8c-4e23-b847-b1bcf7187605, infoPort=33579, infoSecurePort=0, ipcPort=46647, storageInfo=lv=-57;cid=testClusterID;nsid=1749253065;c=1732446851790), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T11:14:14,521 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca 2024-11-24T11:14:14,583 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/zookeeper_0, clientPort=57086, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T11:14:14,592 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57086 2024-11-24T11:14:14,601 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T11:14:14,604 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T11:14:14,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741825_1001 (size=7) 2024-11-24T11:14:15,209 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c with version=8 2024-11-24T11:14:15,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/hbase-staging 2024-11-24T11:14:15,290 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T11:14:15,557 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5f73e76a839:0 server-side Connection retries=45 2024-11-24T11:14:15,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T11:14:15,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T11:14:15,571 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T11:14:15,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T11:14:15,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T11:14:15,700 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T11:14:15,752 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T11:14:15,761 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T11:14:15,764 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T11:14:15,787 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4688 (auto-detected) 2024-11-24T11:14:15,788 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-24T11:14:15,805 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40345 2024-11-24T11:14:15,826 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40345 connecting to ZooKeeper ensemble=127.0.0.1:57086 2024-11-24T11:14:15,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:403450x0, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T11:14:15,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40345-0x1016c8d19510000 connected 2024-11-24T11:14:16,061 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T11:14:16,066 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T11:14:16,075 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T11:14:16,079 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c, hbase.cluster.distributed=false 2024-11-24T11:14:16,104 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T11:14:16,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-24T11:14:16,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40345 2024-11-24T11:14:16,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40345 2024-11-24T11:14:16,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-24T11:14:16,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-24T11:14:16,209 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5f73e76a839:0 server-side Connection retries=45 2024-11-24T11:14:16,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T11:14:16,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T11:14:16,211 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T11:14:16,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T11:14:16,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T11:14:16,214 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T11:14:16,217 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T11:14:16,218 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45939 2024-11-24T11:14:16,220 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45939 connecting to ZooKeeper ensemble=127.0.0.1:57086 2024-11-24T11:14:16,221 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T11:14:16,224 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T11:14:16,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459390x0, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T11:14:16,237 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:459390x0, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T11:14:16,238 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45939-0x1016c8d19510001 connected 2024-11-24T11:14:16,241 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T11:14:16,249 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T11:14:16,251 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T11:14:16,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T11:14:16,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45939 2024-11-24T11:14:16,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45939 2024-11-24T11:14:16,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45939 2024-11-24T11:14:16,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45939 2024-11-24T11:14:16,259 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45939 2024-11-24T11:14:16,275 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5f73e76a839:40345 2024-11-24T11:14:16,276 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5f73e76a839,40345,1732446855400 2024-11-24T11:14:16,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T11:14:16,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T11:14:16,291 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5f73e76a839,40345,1732446855400 2024-11-24T11:14:16,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T11:14:16,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:16,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:16,322 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T11:14:16,323 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5f73e76a839,40345,1732446855400 from backup master directory 2024-11-24T11:14:16,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5f73e76a839,40345,1732446855400 2024-11-24T11:14:16,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T11:14:16,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T11:14:16,332 WARN [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T11:14:16,332 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5f73e76a839,40345,1732446855400 2024-11-24T11:14:16,334 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T11:14:16,336 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T11:14:16,387 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/hbase.id] with ID: d1e11dde-e3cf-4444-a74a-bbf00db082b1 2024-11-24T11:14:16,387 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/.tmp/hbase.id 2024-11-24T11:14:16,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741826_1002 (size=42) 2024-11-24T11:14:16,803 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/.tmp/hbase.id]:[hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/hbase.id] 2024-11-24T11:14:16,851 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T11:14:16,856 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T11:14:16,874 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T11:14:16,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:16,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:16,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741827_1003 (size=196) 2024-11-24T11:14:17,362 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:14:17,364 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T11:14:17,378 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:17,382 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T11:14:17,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741828_1004 (size=1189) 2024-11-24T11:14:17,833 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store 2024-11-24T11:14:17,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741829_1005 (size=34) 2024-11-24T11:14:18,257 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T11:14:18,261 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:18,263 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T11:14:18,263 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:14:18,263 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:14:18,265 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T11:14:18,265 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:14:18,265 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:14:18,266 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732446858262Disabling compacts and flushes for region at 1732446858262Disabling writes for close at 1732446858265 (+3 ms)Writing region close event to WAL at 1732446858265Closed at 1732446858265 2024-11-24T11:14:18,268 WARN [master/f5f73e76a839:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/.initializing 2024-11-24T11:14:18,269 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/WALs/f5f73e76a839,40345,1732446855400 2024-11-24T11:14:18,276 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T11:14:18,289 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5f73e76a839%2C40345%2C1732446855400, suffix=, logDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/WALs/f5f73e76a839,40345,1732446855400, archiveDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/oldWALs, maxLogs=10 2024-11-24T11:14:18,308 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/WALs/f5f73e76a839,40345,1732446855400/f5f73e76a839%2C40345%2C1732446855400.1732446858293, exclude list is [], retry=0 2024-11-24T11:14:18,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34189,DS-6350b056-46ca-4ffe-9891-f23b1a6da2c1,DISK] 2024-11-24T11:14:18,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T11:14:18,359 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/WALs/f5f73e76a839,40345,1732446855400/f5f73e76a839%2C40345%2C1732446855400.1732446858293 2024-11-24T11:14:18,360 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33579:33579)] 2024-11-24T11:14:18,360 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:14:18,361 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:18,364 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,365 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,418 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T11:14:18,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:18,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:18,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T11:14:18,427 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:18,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:18,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T11:14:18,431 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:18,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:18,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T11:14:18,437 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:18,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:18,438 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,442 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,443 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,448 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,449 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,454 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T11:14:18,458 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T11:14:18,463 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:14:18,464 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71661899, jitterRate=0.06784550845623016}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T11:14:18,470 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732446858376Initializing all the Stores at 1732446858379 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732446858379Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446858380 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446858380Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446858380Cleaning up temporary data from old regions at 1732446858450 (+70 ms)Region opened successfully at 1732446858470 (+20 ms) 2024-11-24T11:14:18,471 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T11:14:18,502 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521cbb37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5f73e76a839/172.17.0.3:0 2024-11-24T11:14:18,536 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T11:14:18,545 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T11:14:18,546 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T11:14:18,548 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T11:14:18,549 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T11:14:18,553 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T11:14:18,554 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T11:14:18,576 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T11:14:18,584 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T11:14:18,658 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T11:14:18,663 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T11:14:18,666 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T11:14:18,678 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T11:14:18,680 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T11:14:18,685 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T11:14:18,696 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T11:14:18,698 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T11:14:18,710 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T11:14:18,732 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T11:14:18,741 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T11:14:18,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T11:14:18,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T11:14:18,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:18,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:18,757 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5f73e76a839,40345,1732446855400, sessionid=0x1016c8d19510000, setting cluster-up flag (Was=false) 2024-11-24T11:14:18,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:18,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:18,826 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T11:14:18,829 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5f73e76a839,40345,1732446855400 2024-11-24T11:14:18,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:18,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:18,889 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T11:14:18,894 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5f73e76a839,40345,1732446855400 2024-11-24T11:14:18,903 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T11:14:18,966 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(746): ClusterId : d1e11dde-e3cf-4444-a74a-bbf00db082b1 2024-11-24T11:14:18,968 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T11:14:18,968 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T11:14:18,976 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T11:14:18,982 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T11:14:18,982 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T11:14:18,983 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T11:14:18,987 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5f73e76a839,40345,1732446855400 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T11:14:18,996 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T11:14:18,996 DEBUG [RS:0;f5f73e76a839:45939 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@296f6d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5f73e76a839/172.17.0.3:0 2024-11-24T11:14:18,997 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5f73e76a839:0, corePoolSize=5, maxPoolSize=5 2024-11-24T11:14:18,997 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5f73e76a839:0, corePoolSize=5, maxPoolSize=5 2024-11-24T11:14:18,997 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5f73e76a839:0, corePoolSize=5, maxPoolSize=5 2024-11-24T11:14:18,997 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5f73e76a839:0, corePoolSize=5, maxPoolSize=5 2024-11-24T11:14:18,997 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5f73e76a839:0, corePoolSize=10, maxPoolSize=10 2024-11-24T11:14:18,997 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:18,998 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5f73e76a839:0, corePoolSize=2, maxPoolSize=2 2024-11-24T11:14:18,998 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,001 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732446889001 2024-11-24T11:14:19,003 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T11:14:19,004 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T11:14:19,005 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T11:14:19,006 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T11:14:19,008 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T11:14:19,009 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T11:14:19,009 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T11:14:19,009 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5f73e76a839:45939 2024-11-24T11:14:19,009 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T11:14:19,012 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T11:14:19,010 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,012 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T11:14:19,013 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T11:14:19,013 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:19,013 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T11:14:19,014 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T11:14:19,016 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T11:14:19,016 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5f73e76a839,40345,1732446855400 with port=45939, startcode=1732446856176 2024-11-24T11:14:19,016 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T11:14:19,019 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T11:14:19,020 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T11:14:19,023 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5f73e76a839:0:becomeActiveMaster-HFileCleaner.large.0-1732446859021,5,FailOnTimeoutGroup] 2024-11-24T11:14:19,024 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5f73e76a839:0:becomeActiveMaster-HFileCleaner.small.0-1732446859024,5,FailOnTimeoutGroup] 2024-11-24T11:14:19,025 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,025 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T11:14:19,026 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,027 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741831_1007 (size=1321) 2024-11-24T11:14:19,029 DEBUG [RS:0;f5f73e76a839:45939 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T11:14:19,062 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46679, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T11:14:19,067 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5f73e76a839,45939,1732446856176 2024-11-24T11:14:19,069 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40345 {}] master.ServerManager(517): Registering regionserver=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:19,082 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:14:19,082 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41675 2024-11-24T11:14:19,082 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T11:14:19,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T11:14:19,097 DEBUG [RS:0;f5f73e76a839:45939 {}] zookeeper.ZKUtil(111): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5f73e76a839,45939,1732446856176 2024-11-24T11:14:19,097 WARN [RS:0;f5f73e76a839:45939 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T11:14:19,097 INFO [RS:0;f5f73e76a839:45939 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T11:14:19,098 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/WALs/f5f73e76a839,45939,1732446856176 2024-11-24T11:14:19,099 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5f73e76a839,45939,1732446856176] 2024-11-24T11:14:19,124 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T11:14:19,140 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T11:14:19,145 INFO [RS:0;f5f73e76a839:45939 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T11:14:19,145 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,146 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T11:14:19,151 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T11:14:19,153 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,153 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,153 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,153 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,153 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,153 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,154 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5f73e76a839:0, corePoolSize=2, maxPoolSize=2 2024-11-24T11:14:19,154 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,154 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,154 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,154 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,155 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,155 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5f73e76a839:0, corePoolSize=1, maxPoolSize=1 2024-11-24T11:14:19,155 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5f73e76a839:0, corePoolSize=3, maxPoolSize=3 2024-11-24T11:14:19,155 DEBUG [RS:0;f5f73e76a839:45939 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0, corePoolSize=3, maxPoolSize=3 2024-11-24T11:14:19,157 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,157 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,157 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,157 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,157 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,158 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,45939,1732446856176-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T11:14:19,176 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T11:14:19,177 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,45939,1732446856176-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,178 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,178 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.Replication(171): f5f73e76a839,45939,1732446856176 started 2024-11-24T11:14:19,194 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:19,194 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1482): Serving as f5f73e76a839,45939,1732446856176, RpcServer on f5f73e76a839/172.17.0.3:45939, sessionid=0x1016c8d19510001 2024-11-24T11:14:19,195 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T11:14:19,195 DEBUG [RS:0;f5f73e76a839:45939 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5f73e76a839,45939,1732446856176 2024-11-24T11:14:19,195 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5f73e76a839,45939,1732446856176' 2024-11-24T11:14:19,196 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T11:14:19,197 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T11:14:19,197 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T11:14:19,197 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T11:14:19,198 DEBUG [RS:0;f5f73e76a839:45939 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5f73e76a839,45939,1732446856176 2024-11-24T11:14:19,198 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5f73e76a839,45939,1732446856176' 2024-11-24T11:14:19,198 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T11:14:19,199 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T11:14:19,199 DEBUG [RS:0;f5f73e76a839:45939 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T11:14:19,199 INFO [RS:0;f5f73e76a839:45939 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T11:14:19,200 INFO [RS:0;f5f73e76a839:45939 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T11:14:19,305 INFO [RS:0;f5f73e76a839:45939 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T11:14:19,308 INFO [RS:0;f5f73e76a839:45939 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5f73e76a839%2C45939%2C1732446856176, suffix=, logDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/WALs/f5f73e76a839,45939,1732446856176, archiveDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/oldWALs, maxLogs=32 2024-11-24T11:14:19,324 DEBUG [RS:0;f5f73e76a839:45939 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/WALs/f5f73e76a839,45939,1732446856176/f5f73e76a839%2C45939%2C1732446856176.1732446859312, exclude list is [], retry=0 2024-11-24T11:14:19,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34189,DS-6350b056-46ca-4ffe-9891-f23b1a6da2c1,DISK] 2024-11-24T11:14:19,331 INFO [RS:0;f5f73e76a839:45939 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/WALs/f5f73e76a839,45939,1732446856176/f5f73e76a839%2C45939%2C1732446856176.1732446859312 2024-11-24T11:14:19,332 DEBUG [RS:0;f5f73e76a839:45939 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33579:33579)] 2024-11-24T11:14:19,433 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T11:14:19,434 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:14:19,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741833_1009 (size=32) 2024-11-24T11:14:19,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:19,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T11:14:19,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T11:14:19,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:19,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:19,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T11:14:19,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T11:14:19,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:19,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:19,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T11:14:19,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T11:14:19,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:19,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:19,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T11:14:19,873 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T11:14:19,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:19,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:19,874 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T11:14:19,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740 2024-11-24T11:14:19,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740 2024-11-24T11:14:19,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T11:14:19,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T11:14:19,880 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:14:19,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T11:14:19,886 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:14:19,887 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61236405, jitterRate=-0.08750645816326141}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:14:19,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732446859850Initializing all the Stores at 1732446859855 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732446859855Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732446859855Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446859855Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732446859855Cleaning up temporary data from old regions at 1732446859879 (+24 ms)Region opened successfully at 1732446859889 (+10 ms) 2024-11-24T11:14:19,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T11:14:19,889 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T11:14:19,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T11:14:19,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T11:14:19,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T11:14:19,891 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T11:14:19,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732446859889Disabling compacts and flushes for region at 1732446859889Disabling writes for close at 1732446859890 (+1 ms)Writing region close event to WAL at 1732446859891 (+1 ms)Closed at 1732446859891 2024-11-24T11:14:19,894 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T11:14:19,894 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T11:14:19,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T11:14:19,907 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T11:14:19,910 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T11:14:20,067 DEBUG [f5f73e76a839:40345 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T11:14:20,079 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:20,085 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5f73e76a839,45939,1732446856176, state=OPENING 2024-11-24T11:14:20,142 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T11:14:20,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:20,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:14:20,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T11:14:20,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T11:14:20,156 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T11:14:20,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:14:20,340 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T11:14:20,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33865, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T11:14:20,354 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T11:14:20,355 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T11:14:20,356 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T11:14:20,359 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5f73e76a839%2C45939%2C1732446856176.meta, suffix=.meta, logDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/WALs/f5f73e76a839,45939,1732446856176, archiveDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/oldWALs, maxLogs=32 2024-11-24T11:14:20,376 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/WALs/f5f73e76a839,45939,1732446856176/f5f73e76a839%2C45939%2C1732446856176.meta.1732446860362.meta, exclude list is [], retry=0 2024-11-24T11:14:20,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34189,DS-6350b056-46ca-4ffe-9891-f23b1a6da2c1,DISK] 2024-11-24T11:14:20,383 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/WALs/f5f73e76a839,45939,1732446856176/f5f73e76a839%2C45939%2C1732446856176.meta.1732446860362.meta 2024-11-24T11:14:20,383 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33579:33579)] 2024-11-24T11:14:20,383 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:14:20,385 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T11:14:20,388 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T11:14:20,393 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T11:14:20,397 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T11:14:20,398 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:20,398 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T11:14:20,399 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T11:14:20,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T11:14:20,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T11:14:20,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:20,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:20,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T11:14:20,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T11:14:20,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:20,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:20,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T11:14:20,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T11:14:20,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:20,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:20,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T11:14:20,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T11:14:20,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:20,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T11:14:20,412 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T11:14:20,414 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740 2024-11-24T11:14:20,417 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740 2024-11-24T11:14:20,419 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T11:14:20,419 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T11:14:20,420 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:14:20,423 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T11:14:20,425 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65638840, jitterRate=-0.021905064582824707}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:14:20,426 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T11:14:20,427 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732446860399Writing region info on filesystem at 1732446860399Initializing all the Stores at 1732446860401 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732446860401Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732446860402 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446860402Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732446860402Cleaning up temporary data from old regions at 1732446860419 (+17 ms)Running coprocessor post-open hooks at 1732446860426 (+7 ms)Region opened successfully at 1732446860427 (+1 ms) 2024-11-24T11:14:20,435 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732446860332 2024-11-24T11:14:20,445 DEBUG [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T11:14:20,446 INFO [RS_OPEN_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T11:14:20,447 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:20,449 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5f73e76a839,45939,1732446856176, state=OPEN 2024-11-24T11:14:20,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T11:14:20,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T11:14:20,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T11:14:20,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T11:14:20,488 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:20,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T11:14:20,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5f73e76a839,45939,1732446856176 in 329 msec 2024-11-24T11:14:20,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T11:14:20,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 601 msec 2024-11-24T11:14:20,507 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T11:14:20,507 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T11:14:20,525 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:20,526 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:20,543 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:20,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56707, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:20,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6350 sec 2024-11-24T11:14:20,566 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732446860566, completionTime=-1 2024-11-24T11:14:20,568 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T11:14:20,568 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T11:14:20,617 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T11:14:20,620 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732446920617 2024-11-24T11:14:20,620 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732446980620 2024-11-24T11:14:20,620 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 51 msec 2024-11-24T11:14:20,623 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,40345,1732446855400-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:20,623 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,40345,1732446855400-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:20,623 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,40345,1732446855400-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:20,625 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5f73e76a839:40345, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:20,625 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:20,626 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:20,631 DEBUG [master/f5f73e76a839:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T11:14:20,651 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.318sec 2024-11-24T11:14:20,652 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T11:14:20,653 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T11:14:20,654 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T11:14:20,655 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T11:14:20,655 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T11:14:20,655 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,40345,1732446855400-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T11:14:20,656 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,40345,1732446855400-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T11:14:20,663 DEBUG [master/f5f73e76a839:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T11:14:20,664 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T11:14:20,664 INFO [master/f5f73e76a839:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5f73e76a839,40345,1732446855400-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T11:14:20,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:20,677 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T11:14:20,677 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T11:14:20,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:20,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:20,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72cbfc6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:20,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:20,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:20,700 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:20,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:20,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5f73e76a839,40345,1732446855400 2024-11-24T11:14:20,744 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=6812 2024-11-24T11:14:20,752 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T11:14:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f5f73e76a839,40345,1732446855400 2024-11-24T11:14:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f3f783b 2024-11-24T11:14:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T11:14:20,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T11:14:20,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:14:20,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:14:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:20,778 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T11:14:20,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-24T11:14:20,780 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:20,783 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T11:14:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T11:14:20,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741835_1011 (size=1061) 2024-11-24T11:14:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T11:14:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T11:14:21,199 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:14:21,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741836_1012 (size=53) 2024-11-24T11:14:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T11:14:21,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:21,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 358290acb08925e00ab2bb913fcaac3c, disabling compactions & flushes 2024-11-24T11:14:21,610 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:21,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:21,611 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. after waiting 0 ms 2024-11-24T11:14:21,611 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:21,611 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:21,611 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 358290acb08925e00ab2bb913fcaac3c: Waiting for close lock at 1732446861610Disabling compacts and flushes for region at 1732446861610Disabling writes for close at 1732446861611 (+1 ms)Writing region close event to WAL at 1732446861611Closed at 1732446861611 2024-11-24T11:14:21,615 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T11:14:21,622 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732446861615"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732446861615"}]},"ts":"1732446861615"} 2024-11-24T11:14:21,627 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T11:14:21,629 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T11:14:21,631 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446861629"}]},"ts":"1732446861629"} 2024-11-24T11:14:21,636 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T11:14:21,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, ASSIGN}] 2024-11-24T11:14:21,640 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, ASSIGN 2024-11-24T11:14:21,642 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, ASSIGN; state=OFFLINE, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=false 2024-11-24T11:14:21,794 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=358290acb08925e00ab2bb913fcaac3c, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:21,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, ASSIGN because future has completed 2024-11-24T11:14:21,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:14:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T11:14:21,970 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:21,971 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:14:21,971 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,971 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:21,972 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,972 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,974 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,978 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:21,978 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 358290acb08925e00ab2bb913fcaac3c columnFamilyName A 2024-11-24T11:14:21,979 DEBUG [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:21,980 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.HStore(327): Store=358290acb08925e00ab2bb913fcaac3c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:21,980 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,982 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:21,983 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 358290acb08925e00ab2bb913fcaac3c columnFamilyName B 2024-11-24T11:14:21,983 DEBUG [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:21,984 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.HStore(327): Store=358290acb08925e00ab2bb913fcaac3c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:21,984 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,986 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:21,986 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 358290acb08925e00ab2bb913fcaac3c columnFamilyName C 2024-11-24T11:14:21,987 DEBUG [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:21,987 INFO [StoreOpener-358290acb08925e00ab2bb913fcaac3c-1 {}] regionserver.HStore(327): Store=358290acb08925e00ab2bb913fcaac3c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:21,988 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:21,988 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,990 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,991 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,994 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,994 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:21,995 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:14:21,997 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:22,001 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:14:22,002 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 358290acb08925e00ab2bb913fcaac3c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74671008, jitterRate=0.11268472671508789}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:14:22,002 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:22,003 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 358290acb08925e00ab2bb913fcaac3c: Running coprocessor pre-open hook at 1732446861972Writing region info on filesystem at 1732446861972Initializing all the Stores at 1732446861974 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446861974Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446861974Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446861974Cleaning up temporary data from old regions at 1732446861994 (+20 ms)Running coprocessor post-open hooks at 1732446862002 (+8 ms)Region opened successfully at 1732446862003 (+1 ms) 2024-11-24T11:14:22,006 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., pid=6, masterSystemTime=1732446861959 2024-11-24T11:14:22,009 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:22,010 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:22,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=358290acb08925e00ab2bb913fcaac3c, regionState=OPEN, openSeqNum=2, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:22,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:14:22,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T11:14:22,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 in 215 msec 2024-11-24T11:14:22,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T11:14:22,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, ASSIGN in 386 msec 2024-11-24T11:14:22,030 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T11:14:22,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446862030"}]},"ts":"1732446862030"} 2024-11-24T11:14:22,034 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T11:14:22,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T11:14:22,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2660 sec 2024-11-24T11:14:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T11:14:22,936 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:22,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:22,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:22,947 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6f9c1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:22,950 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685f787d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:22,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:22,959 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T11:14:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f5f73e76a839,40345,1732446855400 2024-11-24T11:14:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c42c849 2024-11-24T11:14:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T11:14:22,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T11:14:22,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:22,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:22,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:22,974 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:22,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:22,978 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:22,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c5f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:22,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:22,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bd1ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:22,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d903666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:22,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:22,994 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396be81f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:22,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d66170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:22,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:22,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:23,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cdbfa6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:23,002 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4097744f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:23,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:23,006 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef82c42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:23,013 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49bfb1d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:23,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:23,018 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e9954f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:23,020 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594b3270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:23,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:23,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc3b165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:23,026 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e8d071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:23,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:23,031 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:23,033 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b32f21b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:23,040 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,040 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,040 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,040 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,040 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,042 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,042 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,043 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,043 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,048 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-24T11:14:23,050 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T11:14:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,064 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:23,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:23,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,070 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,075 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:14:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:23,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:14:23,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:23,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:23,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:23,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:23,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:23,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T11:14:23,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:23,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:23,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/22f1f2f5c70948d0805bfef3fb831ae3 is 50, key is test_row_0/A:col10/1732446863108/Put/seqid=0 2024-11-24T11:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741837_1013 (size=9657) 2024-11-24T11:14:23,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/22f1f2f5c70948d0805bfef3fb831ae3 2024-11-24T11:14:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446923247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446863211, max timestamp=1732446863277Num uniques 859; 2024-11-24T11:14:23,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:14:23,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446863211, max timestamp=1732446863272Num uniques 859; 2024-11-24T11:14:23,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:14:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446863211, max timestamp=1732446863277Num uniques 859; 2024-11-24T11:14:23,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T11:14:23,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446923393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446923388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446923405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446923405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/7ac6d175ba924a8ea96b5a612494d00b is 50, key is test_row_0/B:col10/1732446863108/Put/seqid=0 2024-11-24T11:14:23,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446923435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741838_1014 (size=9657) 2024-11-24T11:14:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/7ac6d175ba924a8ea96b5a612494d00b 2024-11-24T11:14:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446923514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446923514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446923525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446923524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/f9d1a309cb644522976a7769bb74ece7 is 50, key is test_row_0/C:col10/1732446863108/Put/seqid=0 2024-11-24T11:14:23,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741839_1015 (size=9657) 2024-11-24T11:14:23,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/f9d1a309cb644522976a7769bb74ece7 2024-11-24T11:14:23,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/22f1f2f5c70948d0805bfef3fb831ae3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/22f1f2f5c70948d0805bfef3fb831ae3 2024-11-24T11:14:23,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/22f1f2f5c70948d0805bfef3fb831ae3, entries=100, sequenceid=12, filesize=9.4 K 2024-11-24T11:14:23,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/7ac6d175ba924a8ea96b5a612494d00b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7ac6d175ba924a8ea96b5a612494d00b 2024-11-24T11:14:23,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:23,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:23,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7ac6d175ba924a8ea96b5a612494d00b, entries=100, sequenceid=12, filesize=9.4 K 2024-11-24T11:14:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/f9d1a309cb644522976a7769bb74ece7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f9d1a309cb644522976a7769bb74ece7 2024-11-24T11:14:23,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446923646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f9d1a309cb644522976a7769bb74ece7, entries=100, sequenceid=12, filesize=9.4 K 2024-11-24T11:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 358290acb08925e00ab2bb913fcaac3c in 548ms, sequenceid=12, compaction requested=false 2024-11-24T11:14:23,674 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T11:14:23,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T11:14:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:23,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T11:14:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:23,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:23,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:23,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/308cb178472a4b69857c9d57c5bb0e76 is 50, key is test_row_0/A:col10/1732446863379/Put/seqid=0 2024-11-24T11:14:23,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446923760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446923765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446923772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741840_1016 (size=14341) 2024-11-24T11:14:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446923777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/308cb178472a4b69857c9d57c5bb0e76 2024-11-24T11:14:23,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/372dd1fcf2c2459d8fc0e24d1d050443 is 50, key is test_row_0/B:col10/1732446863379/Put/seqid=0 2024-11-24T11:14:23,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741841_1017 (size=12001) 2024-11-24T11:14:23,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/372dd1fcf2c2459d8fc0e24d1d050443 2024-11-24T11:14:23,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446923884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446923895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446923905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446923906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:23,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/4e59b049915942419961f3db89c6db03 is 50, key is test_row_0/C:col10/1732446863379/Put/seqid=0 2024-11-24T11:14:23,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:23,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:23,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741842_1018 (size=12001) 2024-11-24T11:14:23,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446923964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446924094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446924115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446924116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446924125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T11:14:24,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/4e59b049915942419961f3db89c6db03 2024-11-24T11:14:24,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/308cb178472a4b69857c9d57c5bb0e76 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/308cb178472a4b69857c9d57c5bb0e76 2024-11-24T11:14:24,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/308cb178472a4b69857c9d57c5bb0e76, entries=200, sequenceid=49, filesize=14.0 K 2024-11-24T11:14:24,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/372dd1fcf2c2459d8fc0e24d1d050443 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/372dd1fcf2c2459d8fc0e24d1d050443 2024-11-24T11:14:24,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446924404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/372dd1fcf2c2459d8fc0e24d1d050443, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T11:14:24,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/4e59b049915942419961f3db89c6db03 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4e59b049915942419961f3db89c6db03 2024-11-24T11:14:24,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4e59b049915942419961f3db89c6db03, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T11:14:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446924425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 358290acb08925e00ab2bb913fcaac3c in 699ms, sequenceid=49, compaction requested=false 2024-11-24T11:14:24,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:24,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:14:24,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:24,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:24,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:24,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:24,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:24,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:24,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/8be547413b044d45a84e561d33c49034 is 50, key is test_row_0/A:col10/1732446863752/Put/seqid=0 2024-11-24T11:14:24,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741843_1019 (size=12001) 2024-11-24T11:14:24,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/8be547413b044d45a84e561d33c49034 2024-11-24T11:14:24,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b17d29406eca42189fb162ee631736d8 is 50, key is test_row_0/B:col10/1732446863752/Put/seqid=0 2024-11-24T11:14:24,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732446864520, max timestamp=1732446864579Num uniques 859; 2024-11-24T11:14:24,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:24,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446864520, max timestamp=1732446864579Num uniques 859; 2024-11-24T11:14:24,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741844_1020 (size=12001) 2024-11-24T11:14:24,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b17d29406eca42189fb162ee631736d8 2024-11-24T11:14:24,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446864520, max timestamp=1732446864579Num uniques 859; 2024-11-24T11:14:24,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:24,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/21327fd2d8494860bfd83fa26f1514f2 is 50, key is test_row_0/C:col10/1732446863752/Put/seqid=0 2024-11-24T11:14:24,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741845_1021 (size=12001) 2024-11-24T11:14:24,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/21327fd2d8494860bfd83fa26f1514f2 2024-11-24T11:14:24,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446924646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446924649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446924661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/8be547413b044d45a84e561d33c49034 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8be547413b044d45a84e561d33c49034 2024-11-24T11:14:24,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8be547413b044d45a84e561d33c49034, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T11:14:24,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b17d29406eca42189fb162ee631736d8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b17d29406eca42189fb162ee631736d8 2024-11-24T11:14:24,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b17d29406eca42189fb162ee631736d8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T11:14:24,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:24,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:24,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/21327fd2d8494860bfd83fa26f1514f2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/21327fd2d8494860bfd83fa26f1514f2 2024-11-24T11:14:24,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/21327fd2d8494860bfd83fa26f1514f2, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T11:14:24,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 358290acb08925e00ab2bb913fcaac3c in 281ms, sequenceid=61, compaction requested=true 2024-11-24T11:14:24,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:24,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T11:14:24,752 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:24,757 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:24,759 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:24,759 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:24,759 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,760 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7ac6d175ba924a8ea96b5a612494d00b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/372dd1fcf2c2459d8fc0e24d1d050443, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b17d29406eca42189fb162ee631736d8] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=32.9 K 2024-11-24T11:14:24,762 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac6d175ba924a8ea96b5a612494d00b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446863108 2024-11-24T11:14:24,763 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:24,763 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:24,763 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,765 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 372dd1fcf2c2459d8fc0e24d1d050443, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732446863379 2024-11-24T11:14:24,766 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/22f1f2f5c70948d0805bfef3fb831ae3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/308cb178472a4b69857c9d57c5bb0e76, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8be547413b044d45a84e561d33c49034] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=35.2 K 2024-11-24T11:14:24,767 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f1f2f5c70948d0805bfef3fb831ae3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446863108 2024-11-24T11:14:24,768 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b17d29406eca42189fb162ee631736d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732446863752 2024-11-24T11:14:24,770 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 308cb178472a4b69857c9d57c5bb0e76, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732446863379 2024-11-24T11:14:24,771 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be547413b044d45a84e561d33c49034, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732446863752 2024-11-24T11:14:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:24,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T11:14:24,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:24,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:24,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:24,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:24,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:24,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:24,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1dfe8e5fb26c406a958e8eb74272e782 is 50, key is test_row_0/A:col10/1732446864784/Put/seqid=0 2024-11-24T11:14:24,821 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:24,822 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1a484848ba8545d2a80cde9678e907a1 is 50, key is test_row_0/A:col10/1732446863752/Put/seqid=0 2024-11-24T11:14:24,829 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446924815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,831 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/c299660338e54578bbfcff516f35cf67 is 50, key is test_row_0/B:col10/1732446863752/Put/seqid=0 2024-11-24T11:14:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446924828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446924831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741846_1022 (size=14341) 2024-11-24T11:14:24,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1dfe8e5fb26c406a958e8eb74272e782 2024-11-24T11:14:24,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/e73aaf79e354445f8749d519c88a2bdf is 50, key is test_row_0/B:col10/1732446864784/Put/seqid=0 2024-11-24T11:14:24,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:24,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741848_1024 (size=12104) 2024-11-24T11:14:24,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741847_1023 (size=12104) 2024-11-24T11:14:24,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741849_1025 (size=12001) 2024-11-24T11:14:24,922 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1a484848ba8545d2a80cde9678e907a1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1a484848ba8545d2a80cde9678e907a1 2024-11-24T11:14:24,923 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/c299660338e54578bbfcff516f35cf67 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c299660338e54578bbfcff516f35cf67 2024-11-24T11:14:24,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/e73aaf79e354445f8749d519c88a2bdf 2024-11-24T11:14:24,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446924924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446924933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446924934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1271d8b9f1404c56bf85452b028a5579 is 50, key is test_row_0/C:col10/1732446864784/Put/seqid=0 2024-11-24T11:14:24,962 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 1a484848ba8545d2a80cde9678e907a1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:24,962 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into c299660338e54578bbfcff516f35cf67(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:24,962 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:24,962 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:24,965 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446864748; duration=0sec 2024-11-24T11:14:24,966 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:24,966 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:24,967 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:24,967 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446864743; duration=0sec 2024-11-24T11:14:24,967 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:24,967 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:24,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446924955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446924955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:24,975 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:24,975 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:24,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741850_1026 (size=12001) 2024-11-24T11:14:24,977 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:24,978 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f9d1a309cb644522976a7769bb74ece7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4e59b049915942419961f3db89c6db03, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/21327fd2d8494860bfd83fa26f1514f2] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=32.9 K 2024-11-24T11:14:24,979 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d1a309cb644522976a7769bb74ece7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446863108 2024-11-24T11:14:24,981 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e59b049915942419961f3db89c6db03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732446863379 2024-11-24T11:14:24,982 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 21327fd2d8494860bfd83fa26f1514f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732446863752 2024-11-24T11:14:25,009 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:25,010 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/5419460d33c54a5587b2d309fe9987d9 is 50, key is test_row_0/C:col10/1732446863752/Put/seqid=0 2024-11-24T11:14:25,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741851_1027 (size=12104) 2024-11-24T11:14:25,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,047 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/5419460d33c54a5587b2d309fe9987d9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/5419460d33c54a5587b2d309fe9987d9 2024-11-24T11:14:25,069 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into 5419460d33c54a5587b2d309fe9987d9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:25,069 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:25,069 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446864748; duration=0sec 2024-11-24T11:14:25,069 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:25,069 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:25,119 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T11:14:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446925144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446925174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446925174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T11:14:25,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1271d8b9f1404c56bf85452b028a5579 2024-11-24T11:14:25,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1dfe8e5fb26c406a958e8eb74272e782 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1dfe8e5fb26c406a958e8eb74272e782 2024-11-24T11:14:25,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1dfe8e5fb26c406a958e8eb74272e782, entries=200, sequenceid=96, filesize=14.0 K 2024-11-24T11:14:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/e73aaf79e354445f8749d519c88a2bdf as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e73aaf79e354445f8749d519c88a2bdf 2024-11-24T11:14:25,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e73aaf79e354445f8749d519c88a2bdf, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T11:14:25,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1271d8b9f1404c56bf85452b028a5579 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1271d8b9f1404c56bf85452b028a5579 2024-11-24T11:14:25,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1271d8b9f1404c56bf85452b028a5579, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T11:14:25,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 358290acb08925e00ab2bb913fcaac3c in 666ms, sequenceid=96, compaction requested=false 2024-11-24T11:14:25,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:25,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:14:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:25,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:25,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:25,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:25,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:25,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:25,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d119016d6c304ab6b9b95e098ed27167 is 50, key is test_row_0/A:col10/1732446865457/Put/seqid=0 2024-11-24T11:14:25,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446925525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446925534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446925535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741852_1028 (size=14341) 2024-11-24T11:14:25,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d119016d6c304ab6b9b95e098ed27167 2024-11-24T11:14:25,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/876c5e5599934f2ea2b23c080efbf96b is 50, key is test_row_0/B:col10/1732446865457/Put/seqid=0 2024-11-24T11:14:25,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741853_1029 (size=12001) 2024-11-24T11:14:25,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/876c5e5599934f2ea2b23c080efbf96b 2024-11-24T11:14:25,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b95c3c6c2463405fac361a1009ac0bc6 is 50, key is test_row_0/C:col10/1732446865457/Put/seqid=0 2024-11-24T11:14:25,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741854_1030 (size=12001) 2024-11-24T11:14:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446925644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446925655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446925655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446925864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446925864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446925864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,911 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T11:14:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446925943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446925943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:25,953 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:25,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,954 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:25,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:25,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:25,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:25,991 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T11:14:25,992 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T11:14:25,995 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T11:14:25,995 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T11:14:25,995 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T11:14:25,995 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T11:14:25,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T11:14:25,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T11:14:26,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b95c3c6c2463405fac361a1009ac0bc6 2024-11-24T11:14:26,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d119016d6c304ab6b9b95e098ed27167 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d119016d6c304ab6b9b95e098ed27167 2024-11-24T11:14:26,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d119016d6c304ab6b9b95e098ed27167, entries=200, sequenceid=112, filesize=14.0 K 2024-11-24T11:14:26,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/876c5e5599934f2ea2b23c080efbf96b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/876c5e5599934f2ea2b23c080efbf96b 2024-11-24T11:14:26,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/876c5e5599934f2ea2b23c080efbf96b, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T11:14:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b95c3c6c2463405fac361a1009ac0bc6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b95c3c6c2463405fac361a1009ac0bc6 2024-11-24T11:14:26,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b95c3c6c2463405fac361a1009ac0bc6, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T11:14:26,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 358290acb08925e00ab2bb913fcaac3c in 657ms, sequenceid=112, compaction requested=true 2024-11-24T11:14:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:26,119 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T11:14:26,120 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:26,121 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:26,121 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:26,121 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:26,121 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1a484848ba8545d2a80cde9678e907a1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1dfe8e5fb26c406a958e8eb74272e782, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d119016d6c304ab6b9b95e098ed27167] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=39.8 K 2024-11-24T11:14:26,122 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:26,122 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:26,122 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a484848ba8545d2a80cde9678e907a1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732446863752 2024-11-24T11:14:26,122 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:26,122 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c299660338e54578bbfcff516f35cf67, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e73aaf79e354445f8749d519c88a2bdf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/876c5e5599934f2ea2b23c080efbf96b] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=35.3 K 2024-11-24T11:14:26,123 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfe8e5fb26c406a958e8eb74272e782, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446864642 2024-11-24T11:14:26,123 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting c299660338e54578bbfcff516f35cf67, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732446863752 2024-11-24T11:14:26,124 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e73aaf79e354445f8749d519c88a2bdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446864642 2024-11-24T11:14:26,124 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d119016d6c304ab6b9b95e098ed27167, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732446864811 2024-11-24T11:14:26,124 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 876c5e5599934f2ea2b23c080efbf96b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732446864811 2024-11-24T11:14:26,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T11:14:26,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:26,143 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:14:26,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:26,155 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#18 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:26,156 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/44ab84aa6e7a4b9db00db03777cdc3ae is 50, key is test_row_0/B:col10/1732446865457/Put/seqid=0 2024-11-24T11:14:26,167 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#19 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:26,168 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/29b9d362bd3e44b7b76537b36ca89c9d is 50, key is test_row_0/A:col10/1732446865457/Put/seqid=0 2024-11-24T11:14:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/cffb4552c01447beac4c7beb311256f7 is 50, key is test_row_0/A:col10/1732446865529/Put/seqid=0 2024-11-24T11:14:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:26,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:26,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741856_1032 (size=12207) 2024-11-24T11:14:26,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741857_1033 (size=12101) 2024-11-24T11:14:26,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741855_1031 (size=12207) 2024-11-24T11:14:26,188 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/cffb4552c01447beac4c7beb311256f7 2024-11-24T11:14:26,199 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/29b9d362bd3e44b7b76537b36ca89c9d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/29b9d362bd3e44b7b76537b36ca89c9d 2024-11-24T11:14:26,204 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/44ab84aa6e7a4b9db00db03777cdc3ae as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/44ab84aa6e7a4b9db00db03777cdc3ae 2024-11-24T11:14:26,219 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 29b9d362bd3e44b7b76537b36ca89c9d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:26,220 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:26,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f771a9565b7d407495045f4f3b6a7b21 is 50, key is test_row_0/B:col10/1732446865529/Put/seqid=0 2024-11-24T11:14:26,220 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into 44ab84aa6e7a4b9db00db03777cdc3ae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:26,220 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446866118; duration=0sec 2024-11-24T11:14:26,220 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:26,220 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446866119; duration=0sec 2024-11-24T11:14:26,220 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:26,220 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:26,220 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:26,220 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:26,220 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:26,226 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:26,226 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:26,226 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:26,227 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/5419460d33c54a5587b2d309fe9987d9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1271d8b9f1404c56bf85452b028a5579, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b95c3c6c2463405fac361a1009ac0bc6] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=35.3 K 2024-11-24T11:14:26,228 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 5419460d33c54a5587b2d309fe9987d9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732446863752 2024-11-24T11:14:26,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446926219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,230 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1271d8b9f1404c56bf85452b028a5579, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446864642 2024-11-24T11:14:26,231 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b95c3c6c2463405fac361a1009ac0bc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732446864811 2024-11-24T11:14:26,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446926223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446926229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741858_1034 (size=12101) 2024-11-24T11:14:26,254 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:26,256 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/8916584519c140b095d7a6430cd676f2 is 50, key is test_row_0/C:col10/1732446865457/Put/seqid=0 2024-11-24T11:14:26,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f771a9565b7d407495045f4f3b6a7b21 2024-11-24T11:14:26,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/d549924764854cf18d8fabfb33c1b904 is 50, key is test_row_0/C:col10/1732446865529/Put/seqid=0 2024-11-24T11:14:26,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741859_1035 (size=12207) 2024-11-24T11:14:26,313 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/8916584519c140b095d7a6430cd676f2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/8916584519c140b095d7a6430cd676f2 2024-11-24T11:14:26,330 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into 8916584519c140b095d7a6430cd676f2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:26,330 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:26,331 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446866119; duration=0sec 2024-11-24T11:14:26,331 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:26,331 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:26,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741860_1036 (size=12101) 2024-11-24T11:14:26,336 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/d549924764854cf18d8fabfb33c1b904 2024-11-24T11:14:26,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446926335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446926344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446926344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/cffb4552c01447beac4c7beb311256f7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/cffb4552c01447beac4c7beb311256f7 2024-11-24T11:14:26,372 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/cffb4552c01447beac4c7beb311256f7, entries=150, sequenceid=131, filesize=11.8 K 2024-11-24T11:14:26,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f771a9565b7d407495045f4f3b6a7b21 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f771a9565b7d407495045f4f3b6a7b21 2024-11-24T11:14:26,404 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f771a9565b7d407495045f4f3b6a7b21, entries=150, sequenceid=131, filesize=11.8 K 2024-11-24T11:14:26,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/d549924764854cf18d8fabfb33c1b904 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d549924764854cf18d8fabfb33c1b904 2024-11-24T11:14:26,420 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d549924764854cf18d8fabfb33c1b904, entries=150, sequenceid=131, filesize=11.8 K 2024-11-24T11:14:26,423 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 358290acb08925e00ab2bb913fcaac3c in 278ms, sequenceid=131, compaction requested=false 2024-11-24T11:14:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-24T11:14:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-24T11:14:26,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T11:14:26,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3630 sec 2024-11-24T11:14:26,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.3910 sec 2024-11-24T11:14:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:26,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:14:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:26,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/c6389355cf4344dead180439b5de4463 is 50, key is test_row_0/A:col10/1732446866179/Put/seqid=0 2024-11-24T11:14:26,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741861_1037 (size=16931) 2024-11-24T11:14:26,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/c6389355cf4344dead180439b5de4463 2024-11-24T11:14:26,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446926607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/093cb0b03e5e40da98b504ca2403cb46 is 50, key is test_row_0/B:col10/1732446866179/Put/seqid=0 2024-11-24T11:14:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446926621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446926628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741862_1038 (size=12151) 2024-11-24T11:14:26,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446926734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446926734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446926734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446926943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446926944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446926944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/093cb0b03e5e40da98b504ca2403cb46 2024-11-24T11:14:27,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9dba0fef8e3a486b882015558ac98b64 is 50, key is test_row_0/C:col10/1732446866179/Put/seqid=0 2024-11-24T11:14:27,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741863_1039 (size=12151) 2024-11-24T11:14:27,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9dba0fef8e3a486b882015558ac98b64 2024-11-24T11:14:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/c6389355cf4344dead180439b5de4463 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/c6389355cf4344dead180439b5de4463 2024-11-24T11:14:27,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/c6389355cf4344dead180439b5de4463, entries=250, sequenceid=149, filesize=16.5 K 2024-11-24T11:14:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/093cb0b03e5e40da98b504ca2403cb46 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/093cb0b03e5e40da98b504ca2403cb46 2024-11-24T11:14:27,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/093cb0b03e5e40da98b504ca2403cb46, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T11:14:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9dba0fef8e3a486b882015558ac98b64 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9dba0fef8e3a486b882015558ac98b64 2024-11-24T11:14:27,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9dba0fef8e3a486b882015558ac98b64, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T11:14:27,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 358290acb08925e00ab2bb913fcaac3c in 627ms, sequenceid=149, compaction requested=true 2024-11-24T11:14:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:27,186 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:27,186 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:27,188 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:27,188 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:27,188 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,190 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41239 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:27,190 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:27,190 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,191 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/29b9d362bd3e44b7b76537b36ca89c9d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/cffb4552c01447beac4c7beb311256f7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/c6389355cf4344dead180439b5de4463] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=40.3 K 2024-11-24T11:14:27,191 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/44ab84aa6e7a4b9db00db03777cdc3ae, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f771a9565b7d407495045f4f3b6a7b21, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/093cb0b03e5e40da98b504ca2403cb46] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=35.6 K 2024-11-24T11:14:27,192 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b9d362bd3e44b7b76537b36ca89c9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732446864811 2024-11-24T11:14:27,192 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ab84aa6e7a4b9db00db03777cdc3ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732446864811 2024-11-24T11:14:27,193 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f771a9565b7d407495045f4f3b6a7b21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732446865499 2024-11-24T11:14:27,193 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting cffb4552c01447beac4c7beb311256f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732446865499 2024-11-24T11:14:27,194 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 093cb0b03e5e40da98b504ca2403cb46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446866179 2024-11-24T11:14:27,194 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c6389355cf4344dead180439b5de4463, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446866179 2024-11-24T11:14:27,223 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:27,224 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/c6b3fbc7ae614f08b7b7c4dbfdbebd77 is 50, key is test_row_0/B:col10/1732446866179/Put/seqid=0 2024-11-24T11:14:27,232 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:27,233 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/0e4a6cf190f6429c99d7eb66873b6463 is 50, key is test_row_0/A:col10/1732446866179/Put/seqid=0 2024-11-24T11:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T11:14:27,235 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:27,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-24T11:14:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T11:14:27,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:27,241 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:27,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:27,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:14:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:27,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:27,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:27,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:27,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:27,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:27,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741864_1040 (size=12459) 2024-11-24T11:14:27,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741865_1041 (size=12459) 2024-11-24T11:14:27,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/266d2ad523d142618dbd9505c9b37132 is 50, key is test_row_0/A:col10/1732446866616/Put/seqid=0 2024-11-24T11:14:27,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446927283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446927286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446927294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741866_1042 (size=14541) 2024-11-24T11:14:27,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/266d2ad523d142618dbd9505c9b37132 2024-11-24T11:14:27,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/4e06308d8bc440b8b142c0d9a7813f8d is 50, key is test_row_0/B:col10/1732446866616/Put/seqid=0 2024-11-24T11:14:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741867_1043 (size=12151) 2024-11-24T11:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T11:14:27,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446927403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446927404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446927404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:27,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T11:14:27,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446927614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446927614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446927616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,695 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/0e4a6cf190f6429c99d7eb66873b6463 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/0e4a6cf190f6429c99d7eb66873b6463 2024-11-24T11:14:27,696 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/c6b3fbc7ae614f08b7b7c4dbfdbebd77 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c6b3fbc7ae614f08b7b7c4dbfdbebd77 2024-11-24T11:14:27,705 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 0e4a6cf190f6429c99d7eb66873b6463(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:27,705 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:27,705 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446867186; duration=0sec 2024-11-24T11:14:27,705 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:27,705 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:27,705 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:27,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:27,709 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into c6b3fbc7ae614f08b7b7c4dbfdbebd77(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,709 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:27,709 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446867186; duration=0sec 2024-11-24T11:14:27,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,710 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:27,710 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:27,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,711 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:27,711 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:27,711 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,711 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/8916584519c140b095d7a6430cd676f2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d549924764854cf18d8fabfb33c1b904, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9dba0fef8e3a486b882015558ac98b64] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=35.6 K 2024-11-24T11:14:27,713 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8916584519c140b095d7a6430cd676f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732446864811 2024-11-24T11:14:27,714 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d549924764854cf18d8fabfb33c1b904, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732446865499 2024-11-24T11:14:27,715 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dba0fef8e3a486b882015558ac98b64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446866179 2024-11-24T11:14:27,731 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:27,732 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/c8ec03ebdd41418d8a7c8718d042eeff is 50, key is test_row_0/C:col10/1732446866179/Put/seqid=0 2024-11-24T11:14:27,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/4e06308d8bc440b8b142c0d9a7813f8d 2024-11-24T11:14:27,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741868_1044 (size=12459) 2024-11-24T11:14:27,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/fff760d52fe541e48544393a9f5c7960 is 50, key is test_row_0/C:col10/1732446866616/Put/seqid=0 2024-11-24T11:14:27,772 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/c8ec03ebdd41418d8a7c8718d042eeff as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c8ec03ebdd41418d8a7c8718d042eeff 2024-11-24T11:14:27,783 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into c8ec03ebdd41418d8a7c8718d042eeff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:27,783 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:27,783 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446867186; duration=0sec 2024-11-24T11:14:27,784 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:27,785 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741869_1045 (size=12151) 2024-11-24T11:14:27,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/fff760d52fe541e48544393a9f5c7960 2024-11-24T11:14:27,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/266d2ad523d142618dbd9505c9b37132 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/266d2ad523d142618dbd9505c9b37132 2024-11-24T11:14:27,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/266d2ad523d142618dbd9505c9b37132, entries=200, sequenceid=168, filesize=14.2 K 2024-11-24T11:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/4e06308d8bc440b8b142c0d9a7813f8d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e06308d8bc440b8b142c0d9a7813f8d 2024-11-24T11:14:27,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e06308d8bc440b8b142c0d9a7813f8d, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T11:14:27,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/fff760d52fe541e48544393a9f5c7960 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/fff760d52fe541e48544393a9f5c7960 2024-11-24T11:14:27,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/fff760d52fe541e48544393a9f5c7960, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T11:14:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T11:14:27,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:27,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 358290acb08925e00ab2bb913fcaac3c in 609ms, sequenceid=168, compaction requested=false 2024-11-24T11:14:27,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:27,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:14:27,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:27,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:27,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/7caed5c48a1841f58bffbe63268d92ac is 50, key is test_row_0/A:col10/1732446867931/Put/seqid=0 2024-11-24T11:14:27,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741870_1046 (size=14541) 2024-11-24T11:14:27,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/7caed5c48a1841f58bffbe63268d92ac 2024-11-24T11:14:27,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/56f28636aa954733b360f5d0c7118f5f is 50, key is test_row_0/B:col10/1732446867931/Put/seqid=0 2024-11-24T11:14:27,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446927974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446927977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741871_1047 (size=12151) 2024-11-24T11:14:27,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/56f28636aa954733b360f5d0c7118f5f 2024-11-24T11:14:27,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446927981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446927986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:27,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446927983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:27,996 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/2f0bea2cb17e4256b00abdfadeee8dcf is 50, key is test_row_0/C:col10/1732446867931/Put/seqid=0 2024-11-24T11:14:28,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741872_1048 (size=12151) 2024-11-24T11:14:28,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/2f0bea2cb17e4256b00abdfadeee8dcf 2024-11-24T11:14:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/7caed5c48a1841f58bffbe63268d92ac as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/7caed5c48a1841f58bffbe63268d92ac 2024-11-24T11:14:28,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/7caed5c48a1841f58bffbe63268d92ac, entries=200, sequenceid=186, filesize=14.2 K 2024-11-24T11:14:28,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/56f28636aa954733b360f5d0c7118f5f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/56f28636aa954733b360f5d0c7118f5f 2024-11-24T11:14:28,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/56f28636aa954733b360f5d0c7118f5f, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T11:14:28,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/2f0bea2cb17e4256b00abdfadeee8dcf as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2f0bea2cb17e4256b00abdfadeee8dcf 2024-11-24T11:14:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446928084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2f0bea2cb17e4256b00abdfadeee8dcf, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T11:14:28,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 358290acb08925e00ab2bb913fcaac3c in 158ms, sequenceid=186, compaction requested=true 2024-11-24T11:14:28,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:28,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:28,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:28,090 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:28,090 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:28,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:28,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:28,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:28,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:28,092 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:28,092 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:28,092 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:28,092 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:28,093 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,093 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,093 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/0e4a6cf190f6429c99d7eb66873b6463, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/266d2ad523d142618dbd9505c9b37132, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/7caed5c48a1841f58bffbe63268d92ac] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=40.6 K 2024-11-24T11:14:28,093 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c6b3fbc7ae614f08b7b7c4dbfdbebd77, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e06308d8bc440b8b142c0d9a7813f8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/56f28636aa954733b360f5d0c7118f5f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=35.9 K 2024-11-24T11:14:28,094 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4a6cf190f6429c99d7eb66873b6463, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446866179 2024-11-24T11:14:28,094 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b3fbc7ae614f08b7b7c4dbfdbebd77, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446866179 2024-11-24T11:14:28,094 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 266d2ad523d142618dbd9505c9b37132, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732446866602 2024-11-24T11:14:28,095 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e06308d8bc440b8b142c0d9a7813f8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732446866616 2024-11-24T11:14:28,095 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7caed5c48a1841f58bffbe63268d92ac, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446867280 2024-11-24T11:14:28,096 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f28636aa954733b360f5d0c7118f5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446867280 2024-11-24T11:14:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:28,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:14:28,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:28,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:28,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:28,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:28,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:28,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:28,115 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:28,116 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/15d919ed434e4054b3e1da8b8c683870 is 50, key is test_row_0/A:col10/1732446867931/Put/seqid=0 2024-11-24T11:14:28,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/72c9800c2cb442f28c58d7ef0b708c99 is 50, key is test_row_0/A:col10/1732446867944/Put/seqid=0 2024-11-24T11:14:28,131 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:28,132 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/9830ccc06af34582aefbb0918a5e95d7 is 50, key is test_row_0/B:col10/1732446867931/Put/seqid=0 2024-11-24T11:14:28,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446928132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446928134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446928142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741874_1050 (size=12151) 2024-11-24T11:14:28,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741873_1049 (size=12561) 2024-11-24T11:14:28,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:28,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:28,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,175 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/15d919ed434e4054b3e1da8b8c683870 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/15d919ed434e4054b3e1da8b8c683870 2024-11-24T11:14:28,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741875_1051 (size=12561) 2024-11-24T11:14:28,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,185 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/9830ccc06af34582aefbb0918a5e95d7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/9830ccc06af34582aefbb0918a5e95d7 2024-11-24T11:14:28,200 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 15d919ed434e4054b3e1da8b8c683870(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:28,200 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:28,200 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446868090; duration=0sec 2024-11-24T11:14:28,200 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:28,200 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:28,201 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:28,203 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into 9830ccc06af34582aefbb0918a5e95d7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:28,203 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:28,203 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446868090; duration=0sec 2024-11-24T11:14:28,203 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:28,203 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:28,206 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:28,206 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:28,206 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,206 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c8ec03ebdd41418d8a7c8718d042eeff, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/fff760d52fe541e48544393a9f5c7960, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2f0bea2cb17e4256b00abdfadeee8dcf] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=35.9 K 2024-11-24T11:14:28,207 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8ec03ebdd41418d8a7c8718d042eeff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446866179 2024-11-24T11:14:28,208 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting fff760d52fe541e48544393a9f5c7960, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732446866616 2024-11-24T11:14:28,209 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f0bea2cb17e4256b00abdfadeee8dcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446867280 2024-11-24T11:14:28,225 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:28,226 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/4f6628dff02c468180c7683b341155a2 is 50, key is test_row_0/C:col10/1732446867931/Put/seqid=0 2024-11-24T11:14:28,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741876_1052 (size=12561) 2024-11-24T11:14:28,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446928243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,253 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/4f6628dff02c468180c7683b341155a2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4f6628dff02c468180c7683b341155a2 2024-11-24T11:14:28,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446928254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446928255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,275 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into 4f6628dff02c468180c7683b341155a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:28,275 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:28,275 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446868090; duration=0sec 2024-11-24T11:14:28,276 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:28,276 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:28,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446928295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:28,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T11:14:28,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446928464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446928464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446928464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/72c9800c2cb442f28c58d7ef0b708c99 2024-11-24T11:14:28,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/2ce7cfadcdda48ba96042c675d49e4b2 is 50, key is test_row_0/B:col10/1732446867944/Put/seqid=0 2024-11-24T11:14:28,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741877_1053 (size=12151) 2024-11-24T11:14:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446928613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446928774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446928774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446928774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:28,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:28,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:28,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/2ce7cfadcdda48ba96042c675d49e4b2 2024-11-24T11:14:29,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/6942bc5d349a42e8927157d5bda5fdb5 is 50, key is test_row_0/C:col10/1732446867944/Put/seqid=0 2024-11-24T11:14:29,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741878_1054 (size=12151) 2024-11-24T11:14:29,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/6942bc5d349a42e8927157d5bda5fdb5 2024-11-24T11:14:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/72c9800c2cb442f28c58d7ef0b708c99 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/72c9800c2cb442f28c58d7ef0b708c99 2024-11-24T11:14:29,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/72c9800c2cb442f28c58d7ef0b708c99, entries=150, sequenceid=205, filesize=11.9 K 2024-11-24T11:14:29,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/2ce7cfadcdda48ba96042c675d49e4b2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/2ce7cfadcdda48ba96042c675d49e4b2 2024-11-24T11:14:29,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/2ce7cfadcdda48ba96042c675d49e4b2, entries=150, sequenceid=205, filesize=11.9 K 2024-11-24T11:14:29,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/6942bc5d349a42e8927157d5bda5fdb5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/6942bc5d349a42e8927157d5bda5fdb5 2024-11-24T11:14:29,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/6942bc5d349a42e8927157d5bda5fdb5, entries=150, sequenceid=205, filesize=11.9 K 2024-11-24T11:14:29,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 358290acb08925e00ab2bb913fcaac3c in 983ms, sequenceid=205, compaction requested=false 2024-11-24T11:14:29,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:29,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T11:14:29,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:29,108 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T11:14:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:29,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d5abbf71d4e74c298d733605ce7c6f45 is 50, key is test_row_0/A:col10/1732446868138/Put/seqid=0 2024-11-24T11:14:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:29,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741879_1055 (size=12151) 2024-11-24T11:14:29,153 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d5abbf71d4e74c298d733605ce7c6f45 2024-11-24T11:14:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/12123cad6e6444eaa7639926b5c18e69 is 50, key is test_row_0/B:col10/1732446868138/Put/seqid=0 2024-11-24T11:14:29,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741880_1056 (size=12151) 2024-11-24T11:14:29,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446929214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:29,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446929284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:29,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446929284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:29,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446929293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:29,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446929324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T11:14:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446929534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:29,596 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/12123cad6e6444eaa7639926b5c18e69 2024-11-24T11:14:29,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/2fe75b1d1add46eb8c039e912946c6df is 50, key is test_row_0/C:col10/1732446868138/Put/seqid=0 2024-11-24T11:14:29,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741881_1057 (size=12151) 2024-11-24T11:14:29,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446929843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,016 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/2fe75b1d1add46eb8c039e912946c6df 2024-11-24T11:14:30,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d5abbf71d4e74c298d733605ce7c6f45 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d5abbf71d4e74c298d733605ce7c6f45 2024-11-24T11:14:30,032 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d5abbf71d4e74c298d733605ce7c6f45, entries=150, sequenceid=221, filesize=11.9 K 2024-11-24T11:14:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/12123cad6e6444eaa7639926b5c18e69 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/12123cad6e6444eaa7639926b5c18e69 2024-11-24T11:14:30,047 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/12123cad6e6444eaa7639926b5c18e69, entries=150, sequenceid=221, filesize=11.9 K 2024-11-24T11:14:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/2fe75b1d1add46eb8c039e912946c6df as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2fe75b1d1add46eb8c039e912946c6df 2024-11-24T11:14:30,061 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2fe75b1d1add46eb8c039e912946c6df, entries=150, sequenceid=221, filesize=11.9 K 2024-11-24T11:14:30,063 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 358290acb08925e00ab2bb913fcaac3c in 956ms, sequenceid=221, compaction requested=true 2024-11-24T11:14:30,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:30,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:30,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-24T11:14:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-24T11:14:30,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T11:14:30,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8280 sec 2024-11-24T11:14:30,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.8390 sec 2024-11-24T11:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:30,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T11:14:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:30,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a8f14d9a4ada448b8c0af57f3645e92e is 50, key is test_row_0/A:col10/1732446869195/Put/seqid=0 2024-11-24T11:14:30,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741882_1058 (size=12151) 2024-11-24T11:14:30,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a8f14d9a4ada448b8c0af57f3645e92e 2024-11-24T11:14:30,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446930327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446930329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446930330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/67386cf7ad5d4b23bec9fd0bc5a7001c is 50, key is test_row_0/B:col10/1732446869195/Put/seqid=0 2024-11-24T11:14:30,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741883_1059 (size=12151) 2024-11-24T11:14:30,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/67386cf7ad5d4b23bec9fd0bc5a7001c 2024-11-24T11:14:30,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446930354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/a3816805ea7b4615bc5fa2dfad755695 is 50, key is test_row_0/C:col10/1732446869195/Put/seqid=0 2024-11-24T11:14:30,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741884_1060 (size=12151) 2024-11-24T11:14:30,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446930434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446930443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446930443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446930643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446930654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446930654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:30,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/a3816805ea7b4615bc5fa2dfad755695 2024-11-24T11:14:30,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a8f14d9a4ada448b8c0af57f3645e92e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a8f14d9a4ada448b8c0af57f3645e92e 2024-11-24T11:14:30,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a8f14d9a4ada448b8c0af57f3645e92e, entries=150, sequenceid=241, filesize=11.9 K 2024-11-24T11:14:30,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/67386cf7ad5d4b23bec9fd0bc5a7001c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/67386cf7ad5d4b23bec9fd0bc5a7001c 2024-11-24T11:14:30,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/67386cf7ad5d4b23bec9fd0bc5a7001c, entries=150, sequenceid=241, filesize=11.9 K 2024-11-24T11:14:30,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/a3816805ea7b4615bc5fa2dfad755695 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/a3816805ea7b4615bc5fa2dfad755695 2024-11-24T11:14:30,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/a3816805ea7b4615bc5fa2dfad755695, entries=150, sequenceid=241, filesize=11.9 K 2024-11-24T11:14:30,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 358290acb08925e00ab2bb913fcaac3c in 500ms, sequenceid=241, compaction requested=true 2024-11-24T11:14:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:30,808 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:30,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:30,808 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:30,813 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:30,813 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:30,813 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:30,813 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:30,813 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:30,813 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:30,813 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/15d919ed434e4054b3e1da8b8c683870, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/72c9800c2cb442f28c58d7ef0b708c99, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d5abbf71d4e74c298d733605ce7c6f45, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a8f14d9a4ada448b8c0af57f3645e92e] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=47.9 K 2024-11-24T11:14:30,813 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/9830ccc06af34582aefbb0918a5e95d7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/2ce7cfadcdda48ba96042c675d49e4b2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/12123cad6e6444eaa7639926b5c18e69, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/67386cf7ad5d4b23bec9fd0bc5a7001c] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=47.9 K 2024-11-24T11:14:30,814 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9830ccc06af34582aefbb0918a5e95d7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446867280 2024-11-24T11:14:30,814 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d919ed434e4054b3e1da8b8c683870, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446867280 2024-11-24T11:14:30,814 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce7cfadcdda48ba96042c675d49e4b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732446867944 2024-11-24T11:14:30,814 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72c9800c2cb442f28c58d7ef0b708c99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732446867944 2024-11-24T11:14:30,815 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 12123cad6e6444eaa7639926b5c18e69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732446868120 2024-11-24T11:14:30,815 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5abbf71d4e74c298d733605ce7c6f45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732446868120 2024-11-24T11:14:30,815 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 67386cf7ad5d4b23bec9fd0bc5a7001c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732446869195 2024-11-24T11:14:30,815 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8f14d9a4ada448b8c0af57f3645e92e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732446869195 2024-11-24T11:14:30,829 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:30,829 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:30,830 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/5c26bdfbf54142d8ac7fb4b3533f1675 is 50, key is test_row_0/B:col10/1732446869195/Put/seqid=0 2024-11-24T11:14:30,831 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/f5172f10efc24823ae87b0a063a9ba98 is 50, key is test_row_0/A:col10/1732446869195/Put/seqid=0 2024-11-24T11:14:30,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741885_1061 (size=12697) 2024-11-24T11:14:30,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741886_1062 (size=12697) 2024-11-24T11:14:30,879 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/5c26bdfbf54142d8ac7fb4b3533f1675 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/5c26bdfbf54142d8ac7fb4b3533f1675 2024-11-24T11:14:30,882 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/f5172f10efc24823ae87b0a063a9ba98 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/f5172f10efc24823ae87b0a063a9ba98 2024-11-24T11:14:30,891 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into f5172f10efc24823ae87b0a063a9ba98(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:30,891 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into 5c26bdfbf54142d8ac7fb4b3533f1675(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:30,891 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:30,891 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:30,891 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=12, startTime=1732446870808; duration=0sec 2024-11-24T11:14:30,891 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=12, startTime=1732446870808; duration=0sec 2024-11-24T11:14:30,891 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:30,891 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:30,891 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:30,891 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:30,891 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:30,893 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:30,893 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:30,893 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:30,893 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4f6628dff02c468180c7683b341155a2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/6942bc5d349a42e8927157d5bda5fdb5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2fe75b1d1add46eb8c039e912946c6df, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/a3816805ea7b4615bc5fa2dfad755695] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=47.9 K 2024-11-24T11:14:30,893 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6628dff02c468180c7683b341155a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446867280 2024-11-24T11:14:30,894 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6942bc5d349a42e8927157d5bda5fdb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732446867944 2024-11-24T11:14:30,896 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe75b1d1add46eb8c039e912946c6df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732446868120 2024-11-24T11:14:30,897 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3816805ea7b4615bc5fa2dfad755695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732446869195 2024-11-24T11:14:30,908 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:30,909 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9b55639b4f694dd28dac45774bb38e5c is 50, key is test_row_0/C:col10/1732446869195/Put/seqid=0 2024-11-24T11:14:30,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741887_1063 (size=12697) 2024-11-24T11:14:30,927 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9b55639b4f694dd28dac45774bb38e5c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9b55639b4f694dd28dac45774bb38e5c 2024-11-24T11:14:30,935 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into 9b55639b4f694dd28dac45774bb38e5c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:30,935 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:30,935 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=12, startTime=1732446870808; duration=0sec 2024-11-24T11:14:30,935 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:30,935 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:30,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T11:14:30,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:30,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:30,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:30,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:30,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:30,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:30,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/ec4847b409c14d04b3f0d43f02f250b3 is 50, key is test_row_0/A:col10/1732446870327/Put/seqid=0 2024-11-24T11:14:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741888_1064 (size=12201) 2024-11-24T11:14:31,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/ec4847b409c14d04b3f0d43f02f250b3 2024-11-24T11:14:31,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446930993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446930995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446930998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/cc5c0093a63f478db9a3778d757db110 is 50, key is test_row_0/B:col10/1732446870327/Put/seqid=0 2024-11-24T11:14:31,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741889_1065 (size=12201) 2024-11-24T11:14:31,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446931103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446931114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446931114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446931313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446931324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446931324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446931364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,372 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T11:14:31,394 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:31,394 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-24T11:14:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T11:14:31,397 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:31,398 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:31,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:31,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/cc5c0093a63f478db9a3778d757db110 2024-11-24T11:14:31,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/3d29e4b704e240e496c676398696253a is 50, key is test_row_0/C:col10/1732446870327/Put/seqid=0 2024-11-24T11:14:31,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741890_1066 (size=12201) 2024-11-24T11:14:31,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/3d29e4b704e240e496c676398696253a 2024-11-24T11:14:31,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/ec4847b409c14d04b3f0d43f02f250b3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/ec4847b409c14d04b3f0d43f02f250b3 2024-11-24T11:14:31,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/ec4847b409c14d04b3f0d43f02f250b3, entries=150, sequenceid=257, filesize=11.9 K 2024-11-24T11:14:31,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/cc5c0093a63f478db9a3778d757db110 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/cc5c0093a63f478db9a3778d757db110 2024-11-24T11:14:31,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/cc5c0093a63f478db9a3778d757db110, entries=150, sequenceid=257, filesize=11.9 K 2024-11-24T11:14:31,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/3d29e4b704e240e496c676398696253a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/3d29e4b704e240e496c676398696253a 2024-11-24T11:14:31,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/3d29e4b704e240e496c676398696253a, entries=150, sequenceid=257, filesize=11.9 K 2024-11-24T11:14:31,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 358290acb08925e00ab2bb913fcaac3c in 525ms, sequenceid=257, compaction requested=false 2024-11-24T11:14:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T11:14:31,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T11:14:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:31,553 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:14:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:31,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/add694a4560443d39f40cd0ec1ec2a50 is 50, key is test_row_0/A:col10/1732446870984/Put/seqid=0 2024-11-24T11:14:31,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741891_1067 (size=12301) 2024-11-24T11:14:31,575 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/add694a4560443d39f40cd0ec1ec2a50 2024-11-24T11:14:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/ab27bb98e19448fcbe9a1ef04427f851 is 50, key is test_row_0/B:col10/1732446870984/Put/seqid=0 2024-11-24T11:14:31,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741892_1068 (size=12301) 2024-11-24T11:14:31,600 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/ab27bb98e19448fcbe9a1ef04427f851 2024-11-24T11:14:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b645edc6f58c4eeab3ba11983d7b2977 is 50, key is test_row_0/C:col10/1732446870984/Put/seqid=0 2024-11-24T11:14:31,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741893_1069 (size=12301) 2024-11-24T11:14:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:31,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446931652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446931656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446931658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T11:14:31,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446931764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446931774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446931774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446931974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446931983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:31,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446931994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446932003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,006 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T11:14:32,024 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b645edc6f58c4eeab3ba11983d7b2977 2024-11-24T11:14:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/add694a4560443d39f40cd0ec1ec2a50 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/add694a4560443d39f40cd0ec1ec2a50 2024-11-24T11:14:32,036 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/add694a4560443d39f40cd0ec1ec2a50, entries=150, sequenceid=276, filesize=12.0 K 2024-11-24T11:14:32,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/ab27bb98e19448fcbe9a1ef04427f851 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ab27bb98e19448fcbe9a1ef04427f851 2024-11-24T11:14:32,043 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ab27bb98e19448fcbe9a1ef04427f851, entries=150, sequenceid=276, filesize=12.0 K 2024-11-24T11:14:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b645edc6f58c4eeab3ba11983d7b2977 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b645edc6f58c4eeab3ba11983d7b2977 2024-11-24T11:14:32,050 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b645edc6f58c4eeab3ba11983d7b2977, entries=150, sequenceid=276, filesize=12.0 K 2024-11-24T11:14:32,051 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 358290acb08925e00ab2bb913fcaac3c in 498ms, sequenceid=276, compaction requested=true 2024-11-24T11:14:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-24T11:14:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-24T11:14:32,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T11:14:32,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 654 msec 2024-11-24T11:14:32,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 662 msec 2024-11-24T11:14:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:32,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:14:32,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:32,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:32,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:32,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:32,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:32,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:32,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/23e515300e2c47669d3714886cddbe72 is 50, key is test_row_0/A:col10/1732446871656/Put/seqid=0 2024-11-24T11:14:32,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741894_1070 (size=14741) 2024-11-24T11:14:32,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/23e515300e2c47669d3714886cddbe72 2024-11-24T11:14:32,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/91a7e429b9184a00a16fc236433d4474 is 50, key is test_row_0/B:col10/1732446871656/Put/seqid=0 2024-11-24T11:14:32,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741895_1071 (size=12301) 2024-11-24T11:14:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446932342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446932344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446932344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446932453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446932464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446932464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T11:14:32,534 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:32,534 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-24T11:14:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T11:14:32,536 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:32,537 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:32,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T11:14:32,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446932664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446932673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446932673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T11:14:32,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:32,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/91a7e429b9184a00a16fc236433d4474 2024-11-24T11:14:32,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e9e280251207425bb68a3018caf15c62 is 50, key is test_row_0/C:col10/1732446871656/Put/seqid=0 2024-11-24T11:14:32,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741896_1072 (size=12301) 2024-11-24T11:14:32,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T11:14:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:32,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T11:14:32,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446932974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446932984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446932984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:32,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T11:14:32,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:32,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:32,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:32,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:33,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T11:14:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:33,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T11:14:33,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e9e280251207425bb68a3018caf15c62 2024-11-24T11:14:33,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/23e515300e2c47669d3714886cddbe72 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/23e515300e2c47669d3714886cddbe72 2024-11-24T11:14:33,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/23e515300e2c47669d3714886cddbe72, entries=200, sequenceid=290, filesize=14.4 K 2024-11-24T11:14:33,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/91a7e429b9184a00a16fc236433d4474 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/91a7e429b9184a00a16fc236433d4474 2024-11-24T11:14:33,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/91a7e429b9184a00a16fc236433d4474, entries=150, sequenceid=290, filesize=12.0 K 2024-11-24T11:14:33,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e9e280251207425bb68a3018caf15c62 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e9e280251207425bb68a3018caf15c62 2024-11-24T11:14:33,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e9e280251207425bb68a3018caf15c62, entries=150, sequenceid=290, filesize=12.0 K 2024-11-24T11:14:33,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 358290acb08925e00ab2bb913fcaac3c in 912ms, sequenceid=290, compaction requested=true 2024-11-24T11:14:33,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:33,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:33,199 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:33,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:33,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:33,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:33,199 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:33,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:33,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:33,201 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51940 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:33,201 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49500 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:33,201 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:33,201 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:33,201 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:33,201 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:33,201 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/f5172f10efc24823ae87b0a063a9ba98, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/ec4847b409c14d04b3f0d43f02f250b3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/add694a4560443d39f40cd0ec1ec2a50, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/23e515300e2c47669d3714886cddbe72] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=50.7 K 2024-11-24T11:14:33,201 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/5c26bdfbf54142d8ac7fb4b3533f1675, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/cc5c0093a63f478db9a3778d757db110, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ab27bb98e19448fcbe9a1ef04427f851, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/91a7e429b9184a00a16fc236433d4474] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=48.3 K 2024-11-24T11:14:33,202 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c26bdfbf54142d8ac7fb4b3533f1675, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732446869195 2024-11-24T11:14:33,202 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5172f10efc24823ae87b0a063a9ba98, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732446869195 2024-11-24T11:14:33,202 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting cc5c0093a63f478db9a3778d757db110, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732446870326 2024-11-24T11:14:33,202 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec4847b409c14d04b3f0d43f02f250b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732446870326 2024-11-24T11:14:33,203 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting ab27bb98e19448fcbe9a1ef04427f851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732446870984 2024-11-24T11:14:33,203 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting add694a4560443d39f40cd0ec1ec2a50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732446870984 2024-11-24T11:14:33,204 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a7e429b9184a00a16fc236433d4474, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732446871640 2024-11-24T11:14:33,204 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e515300e2c47669d3714886cddbe72, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732446871640 2024-11-24T11:14:33,215 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:33,215 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/988beea5ee57478589c8f458d277a9ef is 50, key is test_row_0/A:col10/1732446871656/Put/seqid=0 2024-11-24T11:14:33,215 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:33,216 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f606dd868f7d438dabb601a3232bb75f is 50, key is test_row_0/B:col10/1732446871656/Put/seqid=0 2024-11-24T11:14:33,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741897_1073 (size=12983) 2024-11-24T11:14:33,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741898_1074 (size=12983) 2024-11-24T11:14:33,241 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f606dd868f7d438dabb601a3232bb75f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f606dd868f7d438dabb601a3232bb75f 2024-11-24T11:14:33,243 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/988beea5ee57478589c8f458d277a9ef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/988beea5ee57478589c8f458d277a9ef 2024-11-24T11:14:33,251 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into f606dd868f7d438dabb601a3232bb75f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:33,251 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:33,251 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=12, startTime=1732446873199; duration=0sec 2024-11-24T11:14:33,252 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:33,252 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:33,252 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:33,252 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 988beea5ee57478589c8f458d277a9ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:33,252 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:33,252 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=12, startTime=1732446873199; duration=0sec 2024-11-24T11:14:33,253 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:33,253 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:33,253 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49500 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:33,254 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:33,254 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:33,254 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9b55639b4f694dd28dac45774bb38e5c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/3d29e4b704e240e496c676398696253a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b645edc6f58c4eeab3ba11983d7b2977, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e9e280251207425bb68a3018caf15c62] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=48.3 K 2024-11-24T11:14:33,254 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b55639b4f694dd28dac45774bb38e5c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732446869195 2024-11-24T11:14:33,255 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d29e4b704e240e496c676398696253a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732446870326 2024-11-24T11:14:33,255 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b645edc6f58c4eeab3ba11983d7b2977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732446870984 2024-11-24T11:14:33,256 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e9e280251207425bb68a3018caf15c62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732446871640 2024-11-24T11:14:33,266 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:33,267 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/db3a22e0e3b8435b83b590ada40c582e is 50, key is test_row_0/C:col10/1732446871656/Put/seqid=0 2024-11-24T11:14:33,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741899_1075 (size=12983) 2024-11-24T11:14:33,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T11:14:33,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:33,305 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:14:33,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:33,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:33,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d6cb3faf573849159695e30b199ef7cf is 50, key is test_row_0/A:col10/1732446872337/Put/seqid=0 2024-11-24T11:14:33,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741900_1076 (size=12301) 2024-11-24T11:14:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:33,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:33,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446933428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:33,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446933483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:33,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446933494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:33,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446933494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:33,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446933544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T11:14:33,677 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/db3a22e0e3b8435b83b590ada40c582e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/db3a22e0e3b8435b83b590ada40c582e 2024-11-24T11:14:33,683 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into db3a22e0e3b8435b83b590ada40c582e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:33,683 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:33,683 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=12, startTime=1732446873199; duration=0sec 2024-11-24T11:14:33,683 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:33,683 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:33,720 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d6cb3faf573849159695e30b199ef7cf 2024-11-24T11:14:33,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b74a446187b240f1ae286b0e8f380a1c is 50, key is test_row_0/B:col10/1732446872337/Put/seqid=0 2024-11-24T11:14:33,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741901_1077 (size=12301) 2024-11-24T11:14:33,754 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b74a446187b240f1ae286b0e8f380a1c 2024-11-24T11:14:33,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446933754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e3ec0392f74c490099b12738acf67a7d is 50, key is test_row_0/C:col10/1732446872337/Put/seqid=0 2024-11-24T11:14:33,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741902_1078 (size=12301) 2024-11-24T11:14:34,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446934064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,174 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e3ec0392f74c490099b12738acf67a7d 2024-11-24T11:14:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/d6cb3faf573849159695e30b199ef7cf as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d6cb3faf573849159695e30b199ef7cf 2024-11-24T11:14:34,185 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d6cb3faf573849159695e30b199ef7cf, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T11:14:34,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b74a446187b240f1ae286b0e8f380a1c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b74a446187b240f1ae286b0e8f380a1c 2024-11-24T11:14:34,193 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b74a446187b240f1ae286b0e8f380a1c, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T11:14:34,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e3ec0392f74c490099b12738acf67a7d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e3ec0392f74c490099b12738acf67a7d 2024-11-24T11:14:34,200 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e3ec0392f74c490099b12738acf67a7d, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T11:14:34,201 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 358290acb08925e00ab2bb913fcaac3c in 896ms, sequenceid=311, compaction requested=false 2024-11-24T11:14:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-24T11:14:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-24T11:14:34,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T11:14:34,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6690 sec 2024-11-24T11:14:34,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6750 sec 2024-11-24T11:14:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:34,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:14:34,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:34,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:34,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:34,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:34,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:34,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:34,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/2908becb447742a8acbfd8478685d960 is 50, key is test_row_0/A:col10/1732446873403/Put/seqid=0 2024-11-24T11:14:34,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741903_1079 (size=14741) 2024-11-24T11:14:34,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/2908becb447742a8acbfd8478685d960 2024-11-24T11:14:34,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/241e98089f204143b1f8703bf8034cd4 is 50, key is test_row_0/B:col10/1732446873403/Put/seqid=0 2024-11-24T11:14:34,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446934542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446934542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446934556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741904_1080 (size=12301) 2024-11-24T11:14:34,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/241e98089f204143b1f8703bf8034cd4 2024-11-24T11:14:34,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446934574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1dcb0469467f4fa3834ffb73e899460f is 50, key is test_row_0/C:col10/1732446873403/Put/seqid=0 2024-11-24T11:14:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741905_1081 (size=12301) 2024-11-24T11:14:34,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446934664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446934664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446934664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T11:14:34,694 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:34,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-24T11:14:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T11:14:34,697 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:34,698 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:34,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T11:14:34,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T11:14:34,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:34,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:34,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:34,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:34,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:34,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446934874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446934874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446934874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:34,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1dcb0469467f4fa3834ffb73e899460f 2024-11-24T11:14:35,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T11:14:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:35,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/2908becb447742a8acbfd8478685d960 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/2908becb447742a8acbfd8478685d960 2024-11-24T11:14:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T11:14:35,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/2908becb447742a8acbfd8478685d960, entries=200, sequenceid=326, filesize=14.4 K 2024-11-24T11:14:35,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/241e98089f204143b1f8703bf8034cd4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/241e98089f204143b1f8703bf8034cd4 2024-11-24T11:14:35,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/241e98089f204143b1f8703bf8034cd4, entries=150, sequenceid=326, filesize=12.0 K 2024-11-24T11:14:35,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1dcb0469467f4fa3834ffb73e899460f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1dcb0469467f4fa3834ffb73e899460f 2024-11-24T11:14:35,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1dcb0469467f4fa3834ffb73e899460f, entries=150, sequenceid=326, filesize=12.0 K 2024-11-24T11:14:35,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 358290acb08925e00ab2bb913fcaac3c in 553ms, sequenceid=326, compaction requested=true 2024-11-24T11:14:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:35,059 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:35,059 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:35,061 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:35,061 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:35,061 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:35,061 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:35,061 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,061 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,061 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f606dd868f7d438dabb601a3232bb75f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b74a446187b240f1ae286b0e8f380a1c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/241e98089f204143b1f8703bf8034cd4] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=36.7 K 2024-11-24T11:14:35,061 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/988beea5ee57478589c8f458d277a9ef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d6cb3faf573849159695e30b199ef7cf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/2908becb447742a8acbfd8478685d960] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=39.1 K 2024-11-24T11:14:35,062 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988beea5ee57478589c8f458d277a9ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732446871640 2024-11-24T11:14:35,062 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f606dd868f7d438dabb601a3232bb75f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732446871640 2024-11-24T11:14:35,063 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6cb3faf573849159695e30b199ef7cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732446872337 2024-11-24T11:14:35,063 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b74a446187b240f1ae286b0e8f380a1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732446872337 2024-11-24T11:14:35,064 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2908becb447742a8acbfd8478685d960, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446873403 2024-11-24T11:14:35,064 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 241e98089f204143b1f8703bf8034cd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446873403 2024-11-24T11:14:35,083 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:35,084 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/555c6e9c649f4905a272a407b170b75a is 50, key is test_row_0/A:col10/1732446873403/Put/seqid=0 2024-11-24T11:14:35,087 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:35,088 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/bed5e7fc3bb141e69213031f90f3f791 is 50, key is test_row_0/B:col10/1732446873403/Put/seqid=0 2024-11-24T11:14:35,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741906_1082 (size=13085) 2024-11-24T11:14:35,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741907_1083 (size=13085) 2024-11-24T11:14:35,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T11:14:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,166 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/555c6e9c649f4905a272a407b170b75a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/555c6e9c649f4905a272a407b170b75a 2024-11-24T11:14:35,166 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:14:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/4d8a8443c0474ffda367a447b21d9fd6 is 50, key is test_row_0/A:col10/1732446874552/Put/seqid=0 2024-11-24T11:14:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:35,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:35,196 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 555c6e9c649f4905a272a407b170b75a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:35,196 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:35,196 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446875059; duration=0sec 2024-11-24T11:14:35,200 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:35,201 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:35,202 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:35,209 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:35,209 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:35,209 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,210 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/db3a22e0e3b8435b83b590ada40c582e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e3ec0392f74c490099b12738acf67a7d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1dcb0469467f4fa3834ffb73e899460f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=36.7 K 2024-11-24T11:14:35,211 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting db3a22e0e3b8435b83b590ada40c582e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732446871640 2024-11-24T11:14:35,213 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ec0392f74c490099b12738acf67a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732446872337 2024-11-24T11:14:35,213 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dcb0469467f4fa3834ffb73e899460f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446873403 2024-11-24T11:14:35,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741908_1084 (size=12301) 2024-11-24T11:14:35,218 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/4d8a8443c0474ffda367a447b21d9fd6 2024-11-24T11:14:35,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446935224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446935225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446935228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f4b703f4934a42979f21c4a70b2438d2 is 50, key is test_row_0/B:col10/1732446874552/Put/seqid=0 2024-11-24T11:14:35,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,242 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:35,243 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b205431d239c4f7d8fd3d3fa3dbd181c is 50, key is test_row_0/C:col10/1732446873403/Put/seqid=0 2024-11-24T11:14:35,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741909_1085 (size=12301) 2024-11-24T11:14:35,277 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f4b703f4934a42979f21c4a70b2438d2 2024-11-24T11:14:35,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741910_1086 (size=13085) 2024-11-24T11:14:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e8d985e3572a4211a261a696dd97b3ce is 50, key is test_row_0/C:col10/1732446874552/Put/seqid=0 2024-11-24T11:14:35,307 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/b205431d239c4f7d8fd3d3fa3dbd181c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b205431d239c4f7d8fd3d3fa3dbd181c 2024-11-24T11:14:35,319 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into b205431d239c4f7d8fd3d3fa3dbd181c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:35,319 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:35,319 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446875059; duration=0sec 2024-11-24T11:14:35,319 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:35,319 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T11:14:35,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741911_1087 (size=12301) 2024-11-24T11:14:35,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446935344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446935344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446935344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446935554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446935555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446935556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,575 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/bed5e7fc3bb141e69213031f90f3f791 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bed5e7fc3bb141e69213031f90f3f791 2024-11-24T11:14:35,583 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into bed5e7fc3bb141e69213031f90f3f791(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:35,583 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:35,583 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446875059; duration=0sec 2024-11-24T11:14:35,583 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:35,583 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:35,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446935594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,598 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,728 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e8d985e3572a4211a261a696dd97b3ce 2024-11-24T11:14:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/4d8a8443c0474ffda367a447b21d9fd6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/4d8a8443c0474ffda367a447b21d9fd6 2024-11-24T11:14:35,747 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/4d8a8443c0474ffda367a447b21d9fd6, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T11:14:35,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/f4b703f4934a42979f21c4a70b2438d2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f4b703f4934a42979f21c4a70b2438d2 2024-11-24T11:14:35,765 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f4b703f4934a42979f21c4a70b2438d2, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T11:14:35,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e8d985e3572a4211a261a696dd97b3ce as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e8d985e3572a4211a261a696dd97b3ce 2024-11-24T11:14:35,777 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e8d985e3572a4211a261a696dd97b3ce, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T11:14:35,779 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 358290acb08925e00ab2bb913fcaac3c in 612ms, sequenceid=344, compaction requested=false 2024-11-24T11:14:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-24T11:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-24T11:14:35,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T11:14:35,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0840 sec 2024-11-24T11:14:35,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0940 sec 2024-11-24T11:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T11:14:35,834 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:35,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-24T11:14:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T11:14:35,840 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:35,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:35,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:35,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:14:35,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:35,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:35,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:35,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:35,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:35,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:35,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1b4ad15e2bd643a18231f48979fc6d1f is 50, key is test_row_0/A:col10/1732446875216/Put/seqid=0 2024-11-24T11:14:35,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741912_1088 (size=12301) 2024-11-24T11:14:35,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1b4ad15e2bd643a18231f48979fc6d1f 2024-11-24T11:14:35,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b9f361549e1f43c58f9ddc50fcc6fcae is 50, key is test_row_0/B:col10/1732446875216/Put/seqid=0 2024-11-24T11:14:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T11:14:35,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741913_1089 (size=12301) 2024-11-24T11:14:35,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446935959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b9f361549e1f43c58f9ddc50fcc6fcae 2024-11-24T11:14:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446935962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446935963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:35,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9ffdc6259f4d4254a2a79b427161c6b2 is 50, key is test_row_0/C:col10/1732446875216/Put/seqid=0 2024-11-24T11:14:35,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741914_1090 (size=12301) 2024-11-24T11:14:35,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:35,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9ffdc6259f4d4254a2a79b427161c6b2 2024-11-24T11:14:36,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/1b4ad15e2bd643a18231f48979fc6d1f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1b4ad15e2bd643a18231f48979fc6d1f 2024-11-24T11:14:36,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1b4ad15e2bd643a18231f48979fc6d1f, entries=150, sequenceid=363, filesize=12.0 K 2024-11-24T11:14:36,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/b9f361549e1f43c58f9ddc50fcc6fcae as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b9f361549e1f43c58f9ddc50fcc6fcae 2024-11-24T11:14:36,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b9f361549e1f43c58f9ddc50fcc6fcae, entries=150, sequenceid=363, filesize=12.0 K 2024-11-24T11:14:36,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/9ffdc6259f4d4254a2a79b427161c6b2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9ffdc6259f4d4254a2a79b427161c6b2 2024-11-24T11:14:36,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9ffdc6259f4d4254a2a79b427161c6b2, entries=150, sequenceid=363, filesize=12.0 K 2024-11-24T11:14:36,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 358290acb08925e00ab2bb913fcaac3c in 191ms, sequenceid=363, compaction requested=true 2024-11-24T11:14:36,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:36,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:36,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:36,061 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:36,061 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:36,062 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:36,062 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:36,062 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,062 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bed5e7fc3bb141e69213031f90f3f791, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f4b703f4934a42979f21c4a70b2438d2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b9f361549e1f43c58f9ddc50fcc6fcae] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=36.8 K 2024-11-24T11:14:36,062 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:36,062 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:36,063 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,063 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/555c6e9c649f4905a272a407b170b75a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/4d8a8443c0474ffda367a447b21d9fd6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1b4ad15e2bd643a18231f48979fc6d1f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=36.8 K 2024-11-24T11:14:36,063 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting bed5e7fc3bb141e69213031f90f3f791, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446873403 2024-11-24T11:14:36,063 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 555c6e9c649f4905a272a407b170b75a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446873403 2024-11-24T11:14:36,064 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b703f4934a42979f21c4a70b2438d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732446874533 2024-11-24T11:14:36,064 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8a8443c0474ffda367a447b21d9fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732446874533 2024-11-24T11:14:36,065 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b4ad15e2bd643a18231f48979fc6d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732446875216 2024-11-24T11:14:36,065 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b9f361549e1f43c58f9ddc50fcc6fcae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732446875216 2024-11-24T11:14:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:36,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:14:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:36,082 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:36,083 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/bb369bc55bd24473883c579f13e1e143 is 50, key is test_row_0/B:col10/1732446875216/Put/seqid=0 2024-11-24T11:14:36,094 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:36,095 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a4d382b7f28e40cfa4ca08470cc51c16 is 50, key is test_row_0/A:col10/1732446875216/Put/seqid=0 2024-11-24T11:14:36,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/475addb09bb44683a460b2223e560eeb is 50, key is test_row_0/A:col10/1732446876079/Put/seqid=0 2024-11-24T11:14:36,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446936115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446936118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446936121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741915_1091 (size=13187) 2024-11-24T11:14:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T11:14:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,172 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/bb369bc55bd24473883c579f13e1e143 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bb369bc55bd24473883c579f13e1e143 2024-11-24T11:14:36,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741916_1092 (size=13187) 2024-11-24T11:14:36,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741917_1093 (size=14741) 2024-11-24T11:14:36,182 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into bb369bc55bd24473883c579f13e1e143(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:36,183 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:36,183 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446876061; duration=0sec 2024-11-24T11:14:36,183 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:36,183 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:36,183 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:36,184 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:36,184 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:36,184 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,185 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b205431d239c4f7d8fd3d3fa3dbd181c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e8d985e3572a4211a261a696dd97b3ce, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9ffdc6259f4d4254a2a79b427161c6b2] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=36.8 K 2024-11-24T11:14:36,185 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b205431d239c4f7d8fd3d3fa3dbd181c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446873403 2024-11-24T11:14:36,186 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d985e3572a4211a261a696dd97b3ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732446874533 2024-11-24T11:14:36,186 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ffdc6259f4d4254a2a79b427161c6b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732446875216 2024-11-24T11:14:36,214 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#81 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:36,214 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/0ecf1c3e2c604e6dbf5a5fb8e71f0c0c is 50, key is test_row_0/C:col10/1732446875216/Put/seqid=0 2024-11-24T11:14:36,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446936224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446936234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446936234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741918_1094 (size=13187) 2024-11-24T11:14:36,281 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/0ecf1c3e2c604e6dbf5a5fb8e71f0c0c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/0ecf1c3e2c604e6dbf5a5fb8e71f0c0c 2024-11-24T11:14:36,306 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into 0ecf1c3e2c604e6dbf5a5fb8e71f0c0c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:36,306 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:36,306 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446876061; duration=0sec 2024-11-24T11:14:36,306 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:36,306 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:36,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446936434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446936454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446936454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T11:14:36,569 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a4d382b7f28e40cfa4ca08470cc51c16 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a4d382b7f28e40cfa4ca08470cc51c16 2024-11-24T11:14:36,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/475addb09bb44683a460b2223e560eeb 2024-11-24T11:14:36,580 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into a4d382b7f28e40cfa4ca08470cc51c16(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:36,581 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:36,581 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446876060; duration=0sec 2024-11-24T11:14:36,581 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:36,581 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:36,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/04887e38c76640c7ac89f255b9c41777 is 50, key is test_row_0/B:col10/1732446876079/Put/seqid=0 2024-11-24T11:14:36,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741919_1095 (size=12301) 2024-11-24T11:14:36,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446936764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446936764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446936764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:36,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:36,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T11:14:37,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/04887e38c76640c7ac89f255b9c41777 2024-11-24T11:14:37,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/750c3ef3f5f54b56a2d04700602d0568 is 50, key is test_row_0/C:col10/1732446876079/Put/seqid=0 2024-11-24T11:14:37,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741920_1096 (size=12301) 2024-11-24T11:14:37,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:37,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:37,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:37,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:37,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:37,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:37,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446937274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446937274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446937274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:37,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:37,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:37,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:37,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:37,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:37,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/750c3ef3f5f54b56a2d04700602d0568 2024-11-24T11:14:37,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/475addb09bb44683a460b2223e560eeb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/475addb09bb44683a460b2223e560eeb 2024-11-24T11:14:37,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/475addb09bb44683a460b2223e560eeb, entries=200, sequenceid=380, filesize=14.4 K 2024-11-24T11:14:37,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/04887e38c76640c7ac89f255b9c41777 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/04887e38c76640c7ac89f255b9c41777 2024-11-24T11:14:37,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/04887e38c76640c7ac89f255b9c41777, entries=150, sequenceid=380, filesize=12.0 K 2024-11-24T11:14:37,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/750c3ef3f5f54b56a2d04700602d0568 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/750c3ef3f5f54b56a2d04700602d0568 2024-11-24T11:14:37,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/750c3ef3f5f54b56a2d04700602d0568, entries=150, sequenceid=380, filesize=12.0 K 2024-11-24T11:14:37,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 358290acb08925e00ab2bb913fcaac3c in 1408ms, sequenceid=380, compaction requested=false 2024-11-24T11:14:37,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:37,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T11:14:37,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:37,557 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:14:37,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:37,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a29bd4ce436343f5bf081bb87d13252c is 50, key is test_row_0/A:col10/1732446876107/Put/seqid=0 2024-11-24T11:14:37,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741921_1097 (size=12301) 2024-11-24T11:14:37,585 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a29bd4ce436343f5bf081bb87d13252c 2024-11-24T11:14:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/a3713673ae084ff38e51da89eb60f530 is 50, key is test_row_0/B:col10/1732446876107/Put/seqid=0 2024-11-24T11:14:37,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741922_1098 (size=12301) 2024-11-24T11:14:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:37,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:37,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446937683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:37,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446937794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T11:14:38,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446938004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:38,010 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/a3713673ae084ff38e51da89eb60f530 2024-11-24T11:14:38,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e0262425e8464bc3a9d94f55a4114121 is 50, key is test_row_0/C:col10/1732446876107/Put/seqid=0 2024-11-24T11:14:38,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741923_1099 (size=12301) 2024-11-24T11:14:38,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446938286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:38,290 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446938293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446938294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:38,296 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:38,298 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446938326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:38,449 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e0262425e8464bc3a9d94f55a4114121 2024-11-24T11:14:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/a29bd4ce436343f5bf081bb87d13252c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a29bd4ce436343f5bf081bb87d13252c 2024-11-24T11:14:38,466 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a29bd4ce436343f5bf081bb87d13252c, entries=150, sequenceid=398, filesize=12.0 K 2024-11-24T11:14:38,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/a3713673ae084ff38e51da89eb60f530 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a3713673ae084ff38e51da89eb60f530 2024-11-24T11:14:38,476 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a3713673ae084ff38e51da89eb60f530, entries=150, sequenceid=398, filesize=12.0 K 2024-11-24T11:14:38,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/e0262425e8464bc3a9d94f55a4114121 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e0262425e8464bc3a9d94f55a4114121 2024-11-24T11:14:38,483 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e0262425e8464bc3a9d94f55a4114121, entries=150, sequenceid=398, filesize=12.0 K 2024-11-24T11:14:38,485 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 358290acb08925e00ab2bb913fcaac3c in 927ms, sequenceid=398, compaction requested=true 2024-11-24T11:14:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-24T11:14:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-24T11:14:38,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T11:14:38,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6440 sec 2024-11-24T11:14:38,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.6550 sec 2024-11-24T11:14:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:38,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:14:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:38,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:38,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/75329b3ce7c34464928d430b09eaf507 is 50, key is test_row_0/A:col10/1732446878834/Put/seqid=0 2024-11-24T11:14:38,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741924_1100 (size=14741) 2024-11-24T11:14:38,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446938887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:38,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446938993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:39,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446939204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:39,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:39,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/75329b3ce7c34464928d430b09eaf507 2024-11-24T11:14:39,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/c56b01d0701342ada1d2a9d03884c7d3 is 50, key is test_row_0/B:col10/1732446878834/Put/seqid=0 2024-11-24T11:14:39,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741925_1101 (size=12301) 2024-11-24T11:14:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446939513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:39,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/c56b01d0701342ada1d2a9d03884c7d3 2024-11-24T11:14:39,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/782e4562f652421c986536b895c8248b is 50, key is test_row_0/C:col10/1732446878834/Put/seqid=0 2024-11-24T11:14:39,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741926_1102 (size=12301) 2024-11-24T11:14:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446940023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T11:14:40,036 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:40,036 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-24T11:14:40,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:40,042 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:40,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T11:14:40,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/782e4562f652421c986536b895c8248b 2024-11-24T11:14:40,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/75329b3ce7c34464928d430b09eaf507 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/75329b3ce7c34464928d430b09eaf507 2024-11-24T11:14:40,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/75329b3ce7c34464928d430b09eaf507, entries=200, sequenceid=416, filesize=14.4 K 2024-11-24T11:14:40,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/c56b01d0701342ada1d2a9d03884c7d3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c56b01d0701342ada1d2a9d03884c7d3 2024-11-24T11:14:40,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c56b01d0701342ada1d2a9d03884c7d3, entries=150, sequenceid=416, filesize=12.0 K 2024-11-24T11:14:40,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/782e4562f652421c986536b895c8248b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/782e4562f652421c986536b895c8248b 2024-11-24T11:14:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T11:14:40,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/782e4562f652421c986536b895c8248b, entries=150, sequenceid=416, filesize=12.0 K 2024-11-24T11:14:40,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 358290acb08925e00ab2bb913fcaac3c in 1320ms, sequenceid=416, compaction requested=true 2024-11-24T11:14:40,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:40,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:40,157 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:40,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:40,157 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:40,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:40,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:40,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:40,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:40,159 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:40,159 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:40,159 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:40,159 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:40,159 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:40,159 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:40,159 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bb369bc55bd24473883c579f13e1e143, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/04887e38c76640c7ac89f255b9c41777, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a3713673ae084ff38e51da89eb60f530, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c56b01d0701342ada1d2a9d03884c7d3] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=48.9 K 2024-11-24T11:14:40,159 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a4d382b7f28e40cfa4ca08470cc51c16, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/475addb09bb44683a460b2223e560eeb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a29bd4ce436343f5bf081bb87d13252c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/75329b3ce7c34464928d430b09eaf507] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=53.7 K 2024-11-24T11:14:40,160 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d382b7f28e40cfa4ca08470cc51c16, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732446875216 2024-11-24T11:14:40,160 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting bb369bc55bd24473883c579f13e1e143, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732446875216 2024-11-24T11:14:40,160 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 04887e38c76640c7ac89f255b9c41777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732446875954 2024-11-24T11:14:40,160 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 475addb09bb44683a460b2223e560eeb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732446875945 2024-11-24T11:14:40,160 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting a29bd4ce436343f5bf081bb87d13252c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732446876099 2024-11-24T11:14:40,160 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a3713673ae084ff38e51da89eb60f530, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732446876099 2024-11-24T11:14:40,161 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75329b3ce7c34464928d430b09eaf507, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732446877633 2024-11-24T11:14:40,163 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c56b01d0701342ada1d2a9d03884c7d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732446877664 2024-11-24T11:14:40,179 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:40,180 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/761445607c414df193723d7f8f2f0697 is 50, key is test_row_0/A:col10/1732446878834/Put/seqid=0 2024-11-24T11:14:40,191 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:40,192 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/a72e89238ba24306aec3e6b03f796f97 is 50, key is test_row_0/B:col10/1732446878834/Put/seqid=0 2024-11-24T11:14:40,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T11:14:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:40,199 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:14:40,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:40,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:40,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:40,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:40,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:40,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:40,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741927_1103 (size=13323) 2024-11-24T11:14:40,214 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/761445607c414df193723d7f8f2f0697 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/761445607c414df193723d7f8f2f0697 2024-11-24T11:14:40,223 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 761445607c414df193723d7f8f2f0697(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:40,223 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:40,223 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=12, startTime=1732446880156; duration=0sec 2024-11-24T11:14:40,223 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:40,223 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:40,223 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:40,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/793c898367a24300bced598264771226 is 50, key is test_row_0/A:col10/1732446878876/Put/seqid=0 2024-11-24T11:14:40,229 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:40,229 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:40,229 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:40,229 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/0ecf1c3e2c604e6dbf5a5fb8e71f0c0c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/750c3ef3f5f54b56a2d04700602d0568, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e0262425e8464bc3a9d94f55a4114121, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/782e4562f652421c986536b895c8248b] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=48.9 K 2024-11-24T11:14:40,232 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ecf1c3e2c604e6dbf5a5fb8e71f0c0c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732446875216 2024-11-24T11:14:40,233 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750c3ef3f5f54b56a2d04700602d0568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732446875954 2024-11-24T11:14:40,233 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0262425e8464bc3a9d94f55a4114121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732446876099 2024-11-24T11:14:40,235 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782e4562f652421c986536b895c8248b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732446877664 2024-11-24T11:14:40,258 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:40,259 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/c3e36931624b47879ec859b5e8b61174 is 50, key is test_row_0/C:col10/1732446878834/Put/seqid=0 2024-11-24T11:14:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741928_1104 (size=13323) 2024-11-24T11:14:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741929_1105 (size=12301) 2024-11-24T11:14:40,270 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/793c898367a24300bced598264771226 2024-11-24T11:14:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/7e9f3c6433d54343aee0e7b87115e02c is 50, key is test_row_0/B:col10/1732446878876/Put/seqid=0 2024-11-24T11:14:40,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741930_1106 (size=13323) 2024-11-24T11:14:40,307 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/c3e36931624b47879ec859b5e8b61174 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c3e36931624b47879ec859b5e8b61174 2024-11-24T11:14:40,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:40,329 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into c3e36931624b47879ec859b5e8b61174(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:40,329 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:40,329 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=12, startTime=1732446880157; duration=0sec 2024-11-24T11:14:40,329 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:40,329 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:40,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741931_1107 (size=12301) 2024-11-24T11:14:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T11:14:40,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446940359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446940361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446940365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446940474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446940474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446940474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,668 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/a72e89238ba24306aec3e6b03f796f97 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a72e89238ba24306aec3e6b03f796f97 2024-11-24T11:14:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T11:14:40,675 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into a72e89238ba24306aec3e6b03f796f97(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:40,675 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:40,675 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=12, startTime=1732446880157; duration=0sec 2024-11-24T11:14:40,676 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:40,676 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446940684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446940684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446940684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:40,738 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/7e9f3c6433d54343aee0e7b87115e02c 2024-11-24T11:14:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/36fdbbcce6684c95a98c7303bf67b8d0 is 50, key is test_row_0/C:col10/1732446878876/Put/seqid=0 2024-11-24T11:14:40,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741932_1108 (size=12301) 2024-11-24T11:14:40,751 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/36fdbbcce6684c95a98c7303bf67b8d0 2024-11-24T11:14:40,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/793c898367a24300bced598264771226 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/793c898367a24300bced598264771226 2024-11-24T11:14:40,762 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/793c898367a24300bced598264771226, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T11:14:40,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/7e9f3c6433d54343aee0e7b87115e02c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7e9f3c6433d54343aee0e7b87115e02c 2024-11-24T11:14:40,771 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7e9f3c6433d54343aee0e7b87115e02c, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T11:14:40,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/36fdbbcce6684c95a98c7303bf67b8d0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/36fdbbcce6684c95a98c7303bf67b8d0 2024-11-24T11:14:40,777 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/36fdbbcce6684c95a98c7303bf67b8d0, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T11:14:40,778 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 358290acb08925e00ab2bb913fcaac3c in 579ms, sequenceid=430, compaction requested=false 2024-11-24T11:14:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-24T11:14:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-24T11:14:40,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T11:14:40,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 737 msec 2024-11-24T11:14:40,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 746 msec 2024-11-24T11:14:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:40,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:14:40,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:40,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:40,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:40,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:40,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:40,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:41,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/08bc86d741d24ec0b2b19b46181243c3 is 50, key is test_row_0/A:col10/1732446880362/Put/seqid=0 2024-11-24T11:14:41,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741933_1109 (size=17181) 2024-11-24T11:14:41,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/08bc86d741d24ec0b2b19b46181243c3 2024-11-24T11:14:41,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446941021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446941022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446941024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/3700f57bf354479b8bdaa9ac924da087 is 50, key is test_row_0/B:col10/1732446880362/Put/seqid=0 2024-11-24T11:14:41,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741934_1110 (size=12301) 2024-11-24T11:14:41,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/3700f57bf354479b8bdaa9ac924da087 2024-11-24T11:14:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1732446941043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,045 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 358290acb08925e00ab2bb913fcaac3c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/f5f6f78272724ff081f069df8d6aec67 is 50, key is test_row_0/C:col10/1732446880362/Put/seqid=0 2024-11-24T11:14:41,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741935_1111 (size=12301) 2024-11-24T11:14:41,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446941134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446941134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446941134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T11:14:41,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:41,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-24T11:14:41,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T11:14:41,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:41,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T11:14:41,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T11:14:41,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:41,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:41,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:41,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:41,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:41,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446941344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446941344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446941344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/f5f6f78272724ff081f069df8d6aec67 2024-11-24T11:14:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/08bc86d741d24ec0b2b19b46181243c3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/08bc86d741d24ec0b2b19b46181243c3 2024-11-24T11:14:41,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/08bc86d741d24ec0b2b19b46181243c3, entries=250, sequenceid=453, filesize=16.8 K 2024-11-24T11:14:41,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/3700f57bf354479b8bdaa9ac924da087 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/3700f57bf354479b8bdaa9ac924da087 2024-11-24T11:14:41,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/3700f57bf354479b8bdaa9ac924da087, entries=150, sequenceid=453, filesize=12.0 K 2024-11-24T11:14:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/f5f6f78272724ff081f069df8d6aec67 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f5f6f78272724ff081f069df8d6aec67 2024-11-24T11:14:41,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f5f6f78272724ff081f069df8d6aec67, entries=150, sequenceid=453, filesize=12.0 K 2024-11-24T11:14:41,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 358290acb08925e00ab2bb913fcaac3c in 485ms, sequenceid=453, compaction requested=true 2024-11-24T11:14:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:41,481 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:41,481 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:41,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:41,482 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:41,482 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:41,482 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:41,483 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/761445607c414df193723d7f8f2f0697, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/793c898367a24300bced598264771226, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/08bc86d741d24ec0b2b19b46181243c3] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=41.8 K 2024-11-24T11:14:41,483 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761445607c414df193723d7f8f2f0697, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732446877664 2024-11-24T11:14:41,483 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:41,483 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:41,483 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:41,483 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a72e89238ba24306aec3e6b03f796f97, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7e9f3c6433d54343aee0e7b87115e02c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/3700f57bf354479b8bdaa9ac924da087] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=37.0 K 2024-11-24T11:14:41,484 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 793c898367a24300bced598264771226, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732446878876 2024-11-24T11:14:41,484 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a72e89238ba24306aec3e6b03f796f97, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732446877664 2024-11-24T11:14:41,484 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9f3c6433d54343aee0e7b87115e02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732446878876 2024-11-24T11:14:41,485 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3700f57bf354479b8bdaa9ac924da087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732446880346 2024-11-24T11:14:41,486 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08bc86d741d24ec0b2b19b46181243c3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732446880320 2024-11-24T11:14:41,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T11:14:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:41,494 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:14:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:41,502 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:41,503 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/daafeb0a21d5439a95bbf2b613bf7419 is 50, key is test_row_0/B:col10/1732446880362/Put/seqid=0 2024-11-24T11:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T11:14:41,504 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:41,504 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/47d94d14f968477697f3832cc7de721f is 50, key is test_row_0/A:col10/1732446880362/Put/seqid=0 2024-11-24T11:14:41,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/e05495ddece44893b9f74a36cc35f193 is 50, key is test_row_0/A:col10/1732446881022/Put/seqid=0 2024-11-24T11:14:41,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741936_1112 (size=13425) 2024-11-24T11:14:41,528 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/daafeb0a21d5439a95bbf2b613bf7419 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/daafeb0a21d5439a95bbf2b613bf7419 2024-11-24T11:14:41,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741937_1113 (size=13425) 2024-11-24T11:14:41,535 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into daafeb0a21d5439a95bbf2b613bf7419(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:41,535 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:41,535 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446881481; duration=0sec 2024-11-24T11:14:41,535 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:41,535 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:41,535 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:41,537 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:41,537 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:41,537 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:41,538 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c3e36931624b47879ec859b5e8b61174, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/36fdbbcce6684c95a98c7303bf67b8d0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f5f6f78272724ff081f069df8d6aec67] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=37.0 K 2024-11-24T11:14:41,538 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e36931624b47879ec859b5e8b61174, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732446877664 2024-11-24T11:14:41,539 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/47d94d14f968477697f3832cc7de721f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/47d94d14f968477697f3832cc7de721f 2024-11-24T11:14:41,539 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 36fdbbcce6684c95a98c7303bf67b8d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732446878876 2024-11-24T11:14:41,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741938_1114 (size=12301) 2024-11-24T11:14:41,542 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f5f6f78272724ff081f069df8d6aec67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732446880346 2024-11-24T11:14:41,547 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 47d94d14f968477697f3832cc7de721f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:41,547 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:41,547 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446881481; duration=0sec 2024-11-24T11:14:41,547 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:41,547 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:41,552 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:41,552 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/d47379304a9a4942b6dd869a44ae79b2 is 50, key is test_row_0/C:col10/1732446880362/Put/seqid=0 2024-11-24T11:14:41,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741939_1115 (size=13425) 2024-11-24T11:14:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:41,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:41,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446941720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446941721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446941722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T11:14:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446941834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446941834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446941834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:41,943 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/e05495ddece44893b9f74a36cc35f193 2024-11-24T11:14:41,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/e6e9912a27a048078c05d35d919f152b is 50, key is test_row_0/B:col10/1732446881022/Put/seqid=0 2024-11-24T11:14:41,964 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/d47379304a9a4942b6dd869a44ae79b2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d47379304a9a4942b6dd869a44ae79b2 2024-11-24T11:14:41,971 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into d47379304a9a4942b6dd869a44ae79b2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:41,971 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:41,971 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446881481; duration=0sec 2024-11-24T11:14:41,971 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:41,971 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:41,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741940_1116 (size=12301) 2024-11-24T11:14:41,975 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/e6e9912a27a048078c05d35d919f152b 2024-11-24T11:14:41,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/284eca873345465faf5c80104d538272 is 50, key is test_row_0/C:col10/1732446881022/Put/seqid=0 2024-11-24T11:14:41,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741941_1117 (size=12301) 2024-11-24T11:14:41,993 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/284eca873345465faf5c80104d538272 2024-11-24T11:14:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/e05495ddece44893b9f74a36cc35f193 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/e05495ddece44893b9f74a36cc35f193 2024-11-24T11:14:42,004 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/e05495ddece44893b9f74a36cc35f193, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T11:14:42,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/e6e9912a27a048078c05d35d919f152b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e6e9912a27a048078c05d35d919f152b 2024-11-24T11:14:42,012 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e6e9912a27a048078c05d35d919f152b, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T11:14:42,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/284eca873345465faf5c80104d538272 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/284eca873345465faf5c80104d538272 2024-11-24T11:14:42,020 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/284eca873345465faf5c80104d538272, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T11:14:42,022 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 358290acb08925e00ab2bb913fcaac3c in 528ms, sequenceid=465, compaction requested=false 2024-11-24T11:14:42,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:42,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-24T11:14:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-24T11:14:42,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T11:14:42,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 837 msec 2024-11-24T11:14:42,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 842 msec 2024-11-24T11:14:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:42,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-24T11:14:42,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:42,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446881709, max timestamp=1732446882044Num uniques 859; 2024-11-24T11:14:42,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:42,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:42,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:14:42,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:42,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:42,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:42,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/18f096548c9b4b818cef238ff6c1c545 is 50, key is test_row_0/A:col10/1732446881715/Put/seqid=0 2024-11-24T11:14:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446942067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446942070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446942070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446942072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741942_1118 (size=12301) 2024-11-24T11:14:42,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/18f096548c9b4b818cef238ff6c1c545 2024-11-24T11:14:42,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/0a15f5fb78654926bca7ad36bbfafb5d is 50, key is test_row_0/B:col10/1732446881715/Put/seqid=0 2024-11-24T11:14:42,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741943_1119 (size=12301) 2024-11-24T11:14:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446942183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446942183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446942183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446942183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T11:14:42,324 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:42,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-24T11:14:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T11:14:42,326 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:42,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:42,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:42,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446942393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446942393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446942393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446942394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T11:14:42,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/0a15f5fb78654926bca7ad36bbfafb5d 2024-11-24T11:14:42,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/102b3b18ea7645cab628d9f3360117c6 is 50, key is test_row_0/C:col10/1732446881715/Put/seqid=0 2024-11-24T11:14:42,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741944_1120 (size=12301) 2024-11-24T11:14:42,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:42,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T11:14:42,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38708 deadline: 1732446942703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38722 deadline: 1732446942703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38692 deadline: 1732446942704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38680 deadline: 1732446942704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:14:42,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/102b3b18ea7645cab628d9f3360117c6 2024-11-24T11:14:42,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/18f096548c9b4b818cef238ff6c1c545 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/18f096548c9b4b818cef238ff6c1c545 2024-11-24T11:14:42,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/18f096548c9b4b818cef238ff6c1c545, entries=150, sequenceid=490, filesize=12.0 K 2024-11-24T11:14:42,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/0a15f5fb78654926bca7ad36bbfafb5d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/0a15f5fb78654926bca7ad36bbfafb5d 2024-11-24T11:14:42,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:42,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:42,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:42,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/0a15f5fb78654926bca7ad36bbfafb5d, entries=150, sequenceid=490, filesize=12.0 K 2024-11-24T11:14:42,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/102b3b18ea7645cab628d9f3360117c6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/102b3b18ea7645cab628d9f3360117c6 2024-11-24T11:14:42,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/102b3b18ea7645cab628d9f3360117c6, entries=150, sequenceid=490, filesize=12.0 K 2024-11-24T11:14:42,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for 358290acb08925e00ab2bb913fcaac3c in 902ms, sequenceid=490, compaction requested=true 2024-11-24T11:14:42,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:42,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:42,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:42,950 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:42,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:42,950 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:42,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:42,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 358290acb08925e00ab2bb913fcaac3c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:42,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:42,951 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:42,952 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/B is initiating minor compaction (all files) 2024-11-24T11:14:42,952 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:42,952 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/B in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,952 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/A is initiating minor compaction (all files) 2024-11-24T11:14:42,952 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/A in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,952 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/47d94d14f968477697f3832cc7de721f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/e05495ddece44893b9f74a36cc35f193, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/18f096548c9b4b818cef238ff6c1c545] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=37.1 K 2024-11-24T11:14:42,952 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/daafeb0a21d5439a95bbf2b613bf7419, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e6e9912a27a048078c05d35d919f152b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/0a15f5fb78654926bca7ad36bbfafb5d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=37.1 K 2024-11-24T11:14:42,953 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting daafeb0a21d5439a95bbf2b613bf7419, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732446880346 2024-11-24T11:14:42,953 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d94d14f968477697f3832cc7de721f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732446880346 2024-11-24T11:14:42,953 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e9912a27a048078c05d35d919f152b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732446881011 2024-11-24T11:14:42,953 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e05495ddece44893b9f74a36cc35f193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732446881011 2024-11-24T11:14:42,953 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a15f5fb78654926bca7ad36bbfafb5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732446881709 2024-11-24T11:14:42,954 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18f096548c9b4b818cef238ff6c1c545, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732446881709 2024-11-24T11:14:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T11:14:42,963 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:42,964 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/25814845304649b9a9de424773c2bf73 is 50, key is test_row_0/B:col10/1732446881715/Put/seqid=0 2024-11-24T11:14:42,966 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#A#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:42,967 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/8d58b9eaf0634ee9a2796de97ad6b777 is 50, key is test_row_0/A:col10/1732446881715/Put/seqid=0 2024-11-24T11:14:42,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741945_1121 (size=13527) 2024-11-24T11:14:42,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741946_1122 (size=13527) 2024-11-24T11:14:42,980 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/25814845304649b9a9de424773c2bf73 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/25814845304649b9a9de424773c2bf73 2024-11-24T11:14:42,990 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/8d58b9eaf0634ee9a2796de97ad6b777 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8d58b9eaf0634ee9a2796de97ad6b777 2024-11-24T11:14:42,990 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/B of 358290acb08925e00ab2bb913fcaac3c into 25814845304649b9a9de424773c2bf73(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:42,990 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:42,990 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/B, priority=13, startTime=1732446882950; duration=0sec 2024-11-24T11:14:42,990 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:42,990 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:B 2024-11-24T11:14:42,990 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:42,992 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:42,992 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 358290acb08925e00ab2bb913fcaac3c/C is initiating minor compaction (all files) 2024-11-24T11:14:42,992 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 358290acb08925e00ab2bb913fcaac3c/C in TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:42,993 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d47379304a9a4942b6dd869a44ae79b2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/284eca873345465faf5c80104d538272, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/102b3b18ea7645cab628d9f3360117c6] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp, totalSize=37.1 K 2024-11-24T11:14:42,994 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d47379304a9a4942b6dd869a44ae79b2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732446880346 2024-11-24T11:14:42,995 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 284eca873345465faf5c80104d538272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732446881011 2024-11-24T11:14:42,996 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 102b3b18ea7645cab628d9f3360117c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732446881709 2024-11-24T11:14:42,999 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/A of 358290acb08925e00ab2bb913fcaac3c into 8d58b9eaf0634ee9a2796de97ad6b777(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:43,000 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:43,000 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/A, priority=13, startTime=1732446882950; duration=0sec 2024-11-24T11:14:43,000 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:43,000 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:A 2024-11-24T11:14:43,008 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 358290acb08925e00ab2bb913fcaac3c#C#compaction#110 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:43,009 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1cfd3498d2384c4c8ceef7aff0c393cb is 50, key is test_row_0/C:col10/1732446881715/Put/seqid=0 2024-11-24T11:14:43,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741947_1123 (size=13527) 2024-11-24T11:14:43,028 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/1cfd3498d2384c4c8ceef7aff0c393cb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1cfd3498d2384c4c8ceef7aff0c393cb 2024-11-24T11:14:43,033 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 358290acb08925e00ab2bb913fcaac3c/C of 358290acb08925e00ab2bb913fcaac3c into 1cfd3498d2384c4c8ceef7aff0c393cb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:43,033 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:43,033 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c., storeName=358290acb08925e00ab2bb913fcaac3c/C, priority=13, startTime=1732446882951; duration=0sec 2024-11-24T11:14:43,033 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:43,034 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 358290acb08925e00ab2bb913fcaac3c:C 2024-11-24T11:14:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:43,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:14:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:43,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/647ee123c4be4654af148bd8efaf0dcb is 50, key is test_row_0/A:col10/1732446883055/Put/seqid=0 2024-11-24T11:14:43,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741948_1124 (size=14741) 2024-11-24T11:14:43,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/647ee123c4be4654af148bd8efaf0dcb 2024-11-24T11:14:43,075 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-24T11:14:43,075 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,075 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-24T11:14:43,075 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,076 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-24T11:14:43,076 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/ffa7a27a30384b13932febd897ba80fd is 50, key is test_row_0/B:col10/1732446883055/Put/seqid=0 2024-11-24T11:14:43,081 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,081 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,081 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,081 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-24T11:14:43,081 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,081 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,081 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,081 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,081 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,082 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,085 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-24T11:14:43,085 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,085 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,085 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741949_1125 (size=12301) 2024-11-24T11:14:43,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,218 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-24T11:14:43,218 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-24T11:14:43,218 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,218 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,218 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,218 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,218 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-24T11:14:43,218 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,218 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,218 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,219 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,219 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,220 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-24T11:14:43,220 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:14:43,220 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,220 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:43,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:43,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T11:14:43,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/ffa7a27a30384b13932febd897ba80fd 2024-11-24T11:14:43,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/bce52c858416432a8d60aac2d028d199 is 50, key is test_row_0/C:col10/1732446883055/Put/seqid=0 2024-11-24T11:14:43,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741950_1126 (size=12301) 2024-11-24T11:14:43,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:43,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. as already flushing 2024-11-24T11:14:43,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:43,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:43,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/bce52c858416432a8d60aac2d028d199 2024-11-24T11:14:43,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/647ee123c4be4654af148bd8efaf0dcb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/647ee123c4be4654af148bd8efaf0dcb 2024-11-24T11:14:43,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/647ee123c4be4654af148bd8efaf0dcb, entries=200, sequenceid=509, filesize=14.4 K 2024-11-24T11:14:43,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/ffa7a27a30384b13932febd897ba80fd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ffa7a27a30384b13932febd897ba80fd 2024-11-24T11:14:43,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ffa7a27a30384b13932febd897ba80fd, entries=150, sequenceid=509, filesize=12.0 K 2024-11-24T11:14:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/bce52c858416432a8d60aac2d028d199 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/bce52c858416432a8d60aac2d028d199 2024-11-24T11:14:43,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/bce52c858416432a8d60aac2d028d199, entries=150, sequenceid=509, filesize=12.0 K 2024-11-24T11:14:43,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=60.38 KB/61830 for 358290acb08925e00ab2bb913fcaac3c in 881ms, sequenceid=509, compaction requested=false 2024-11-24T11:14:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:44,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T11:14:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:44,030 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 358290acb08925e00ab2bb913fcaac3c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:14:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=A 2024-11-24T11:14:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=B 2024-11-24T11:14:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 358290acb08925e00ab2bb913fcaac3c, store=C 2024-11-24T11:14:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:44,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/71db110815bd477f98378ef98ba77d97 is 50, key is test_row_0/A:col10/1732446883216/Put/seqid=0 2024-11-24T11:14:44,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741951_1127 (size=12301) 2024-11-24T11:14:44,441 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/71db110815bd477f98378ef98ba77d97 2024-11-24T11:14:44,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/4e4a09a24acf4875a345cc7680b48805 is 50, key is test_row_0/B:col10/1732446883216/Put/seqid=0 2024-11-24T11:14:44,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741952_1128 (size=12301) 2024-11-24T11:14:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T11:14:44,518 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T11:14:44,863 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/4e4a09a24acf4875a345cc7680b48805 2024-11-24T11:14:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/07900e16472c41fab67260ade5bf26fb is 50, key is test_row_0/C:col10/1732446883216/Put/seqid=0 2024-11-24T11:14:44,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741953_1129 (size=12301) 2024-11-24T11:14:45,283 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/07900e16472c41fab67260ade5bf26fb 2024-11-24T11:14:45,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/A/71db110815bd477f98378ef98ba77d97 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/71db110815bd477f98378ef98ba77d97 2024-11-24T11:14:45,339 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/71db110815bd477f98378ef98ba77d97, entries=150, sequenceid=521, filesize=12.0 K 2024-11-24T11:14:45,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/B/4e4a09a24acf4875a345cc7680b48805 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e4a09a24acf4875a345cc7680b48805 2024-11-24T11:14:45,345 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e4a09a24acf4875a345cc7680b48805, entries=150, sequenceid=521, filesize=12.0 K 2024-11-24T11:14:45,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/.tmp/C/07900e16472c41fab67260ade5bf26fb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/07900e16472c41fab67260ade5bf26fb 2024-11-24T11:14:45,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/07900e16472c41fab67260ade5bf26fb, entries=150, sequenceid=521, filesize=12.0 K 2024-11-24T11:14:45,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 358290acb08925e00ab2bb913fcaac3c in 1323ms, sequenceid=521, compaction requested=true 2024-11-24T11:14:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 358290acb08925e00ab2bb913fcaac3c: 2024-11-24T11:14:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-24T11:14:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-24T11:14:45,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T11:14:45,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0280 sec 2024-11-24T11:14:45,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 3.0330 sec 2024-11-24T11:14:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T11:14:46,486 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:46,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T11:14:46,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-24T11:14:46,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-24T11:14:46,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-24T11:14:46,487 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-24T11:14:46,487 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-24T11:14:46,487 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T11:14:46,487 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2690 2024-11-24T11:14:46,487 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2654 2024-11-24T11:14:46,487 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T11:14:46,487 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-24T11:14:46,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3565 rows 2024-11-24T11:14:46,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1190 2024-11-24T11:14:46,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3569 rows 2024-11-24T11:14:46,488 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:14:46,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:14:46,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:46,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:46,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:14:46,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-24T11:14:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T11:14:46,504 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446886504"}]},"ts":"1732446886504"} 2024-11-24T11:14:46,506 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T11:14:46,506 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T11:14:46,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:14:46,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, UNASSIGN}] 2024-11-24T11:14:46,513 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, UNASSIGN 2024-11-24T11:14:46,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=358290acb08925e00ab2bb913fcaac3c, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:46,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, UNASSIGN because future has completed 2024-11-24T11:14:46,516 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:14:46,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:14:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T11:14:46,677 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:46,678 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:14:46,679 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 358290acb08925e00ab2bb913fcaac3c, disabling compactions & flushes 2024-11-24T11:14:46,679 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:46,680 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:46,680 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. after waiting 0 ms 2024-11-24T11:14:46,680 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:46,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/22f1f2f5c70948d0805bfef3fb831ae3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/308cb178472a4b69857c9d57c5bb0e76, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1a484848ba8545d2a80cde9678e907a1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8be547413b044d45a84e561d33c49034, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1dfe8e5fb26c406a958e8eb74272e782, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d119016d6c304ab6b9b95e098ed27167, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/29b9d362bd3e44b7b76537b36ca89c9d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/cffb4552c01447beac4c7beb311256f7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/c6389355cf4344dead180439b5de4463, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/0e4a6cf190f6429c99d7eb66873b6463, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/266d2ad523d142618dbd9505c9b37132, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/7caed5c48a1841f58bffbe63268d92ac, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/15d919ed434e4054b3e1da8b8c683870, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/72c9800c2cb442f28c58d7ef0b708c99, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d5abbf71d4e74c298d733605ce7c6f45, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/f5172f10efc24823ae87b0a063a9ba98, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a8f14d9a4ada448b8c0af57f3645e92e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/ec4847b409c14d04b3f0d43f02f250b3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/add694a4560443d39f40cd0ec1ec2a50, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/23e515300e2c47669d3714886cddbe72, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/988beea5ee57478589c8f458d277a9ef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d6cb3faf573849159695e30b199ef7cf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/2908becb447742a8acbfd8478685d960, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/555c6e9c649f4905a272a407b170b75a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/4d8a8443c0474ffda367a447b21d9fd6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a4d382b7f28e40cfa4ca08470cc51c16, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1b4ad15e2bd643a18231f48979fc6d1f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/475addb09bb44683a460b2223e560eeb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a29bd4ce436343f5bf081bb87d13252c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/75329b3ce7c34464928d430b09eaf507, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/761445607c414df193723d7f8f2f0697, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/793c898367a24300bced598264771226, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/08bc86d741d24ec0b2b19b46181243c3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/47d94d14f968477697f3832cc7de721f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/e05495ddece44893b9f74a36cc35f193, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/18f096548c9b4b818cef238ff6c1c545] to archive 2024-11-24T11:14:46,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:14:46,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/22f1f2f5c70948d0805bfef3fb831ae3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/22f1f2f5c70948d0805bfef3fb831ae3 2024-11-24T11:14:46,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/308cb178472a4b69857c9d57c5bb0e76 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/308cb178472a4b69857c9d57c5bb0e76 2024-11-24T11:14:46,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1a484848ba8545d2a80cde9678e907a1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1a484848ba8545d2a80cde9678e907a1 2024-11-24T11:14:46,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8be547413b044d45a84e561d33c49034 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8be547413b044d45a84e561d33c49034 2024-11-24T11:14:46,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1dfe8e5fb26c406a958e8eb74272e782 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1dfe8e5fb26c406a958e8eb74272e782 2024-11-24T11:14:46,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d119016d6c304ab6b9b95e098ed27167 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d119016d6c304ab6b9b95e098ed27167 2024-11-24T11:14:46,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/29b9d362bd3e44b7b76537b36ca89c9d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/29b9d362bd3e44b7b76537b36ca89c9d 2024-11-24T11:14:46,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/cffb4552c01447beac4c7beb311256f7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/cffb4552c01447beac4c7beb311256f7 2024-11-24T11:14:46,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/c6389355cf4344dead180439b5de4463 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/c6389355cf4344dead180439b5de4463 2024-11-24T11:14:46,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/0e4a6cf190f6429c99d7eb66873b6463 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/0e4a6cf190f6429c99d7eb66873b6463 2024-11-24T11:14:46,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/266d2ad523d142618dbd9505c9b37132 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/266d2ad523d142618dbd9505c9b37132 2024-11-24T11:14:46,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/7caed5c48a1841f58bffbe63268d92ac to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/7caed5c48a1841f58bffbe63268d92ac 2024-11-24T11:14:46,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/15d919ed434e4054b3e1da8b8c683870 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/15d919ed434e4054b3e1da8b8c683870 2024-11-24T11:14:46,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/72c9800c2cb442f28c58d7ef0b708c99 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/72c9800c2cb442f28c58d7ef0b708c99 2024-11-24T11:14:46,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d5abbf71d4e74c298d733605ce7c6f45 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d5abbf71d4e74c298d733605ce7c6f45 2024-11-24T11:14:46,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/f5172f10efc24823ae87b0a063a9ba98 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/f5172f10efc24823ae87b0a063a9ba98 2024-11-24T11:14:46,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a8f14d9a4ada448b8c0af57f3645e92e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a8f14d9a4ada448b8c0af57f3645e92e 2024-11-24T11:14:46,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/ec4847b409c14d04b3f0d43f02f250b3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/ec4847b409c14d04b3f0d43f02f250b3 2024-11-24T11:14:46,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/add694a4560443d39f40cd0ec1ec2a50 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/add694a4560443d39f40cd0ec1ec2a50 2024-11-24T11:14:46,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/23e515300e2c47669d3714886cddbe72 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/23e515300e2c47669d3714886cddbe72 2024-11-24T11:14:46,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/988beea5ee57478589c8f458d277a9ef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/988beea5ee57478589c8f458d277a9ef 2024-11-24T11:14:46,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d6cb3faf573849159695e30b199ef7cf to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/d6cb3faf573849159695e30b199ef7cf 2024-11-24T11:14:46,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/2908becb447742a8acbfd8478685d960 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/2908becb447742a8acbfd8478685d960 2024-11-24T11:14:46,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/555c6e9c649f4905a272a407b170b75a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/555c6e9c649f4905a272a407b170b75a 2024-11-24T11:14:46,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/4d8a8443c0474ffda367a447b21d9fd6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/4d8a8443c0474ffda367a447b21d9fd6 2024-11-24T11:14:46,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a4d382b7f28e40cfa4ca08470cc51c16 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a4d382b7f28e40cfa4ca08470cc51c16 2024-11-24T11:14:46,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1b4ad15e2bd643a18231f48979fc6d1f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/1b4ad15e2bd643a18231f48979fc6d1f 2024-11-24T11:14:46,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/475addb09bb44683a460b2223e560eeb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/475addb09bb44683a460b2223e560eeb 2024-11-24T11:14:46,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a29bd4ce436343f5bf081bb87d13252c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/a29bd4ce436343f5bf081bb87d13252c 2024-11-24T11:14:46,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/75329b3ce7c34464928d430b09eaf507 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/75329b3ce7c34464928d430b09eaf507 2024-11-24T11:14:46,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/761445607c414df193723d7f8f2f0697 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/761445607c414df193723d7f8f2f0697 2024-11-24T11:14:46,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/793c898367a24300bced598264771226 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/793c898367a24300bced598264771226 2024-11-24T11:14:46,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/08bc86d741d24ec0b2b19b46181243c3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/08bc86d741d24ec0b2b19b46181243c3 2024-11-24T11:14:46,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/47d94d14f968477697f3832cc7de721f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/47d94d14f968477697f3832cc7de721f 2024-11-24T11:14:46,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/e05495ddece44893b9f74a36cc35f193 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/e05495ddece44893b9f74a36cc35f193 2024-11-24T11:14:46,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/18f096548c9b4b818cef238ff6c1c545 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/18f096548c9b4b818cef238ff6c1c545 2024-11-24T11:14:46,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7ac6d175ba924a8ea96b5a612494d00b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/372dd1fcf2c2459d8fc0e24d1d050443, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c299660338e54578bbfcff516f35cf67, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b17d29406eca42189fb162ee631736d8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e73aaf79e354445f8749d519c88a2bdf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/44ab84aa6e7a4b9db00db03777cdc3ae, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/876c5e5599934f2ea2b23c080efbf96b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f771a9565b7d407495045f4f3b6a7b21, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c6b3fbc7ae614f08b7b7c4dbfdbebd77, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/093cb0b03e5e40da98b504ca2403cb46, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e06308d8bc440b8b142c0d9a7813f8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/9830ccc06af34582aefbb0918a5e95d7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/56f28636aa954733b360f5d0c7118f5f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/2ce7cfadcdda48ba96042c675d49e4b2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/12123cad6e6444eaa7639926b5c18e69, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/5c26bdfbf54142d8ac7fb4b3533f1675, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/67386cf7ad5d4b23bec9fd0bc5a7001c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/cc5c0093a63f478db9a3778d757db110, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ab27bb98e19448fcbe9a1ef04427f851, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f606dd868f7d438dabb601a3232bb75f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/91a7e429b9184a00a16fc236433d4474, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b74a446187b240f1ae286b0e8f380a1c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bed5e7fc3bb141e69213031f90f3f791, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/241e98089f204143b1f8703bf8034cd4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f4b703f4934a42979f21c4a70b2438d2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bb369bc55bd24473883c579f13e1e143, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b9f361549e1f43c58f9ddc50fcc6fcae, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/04887e38c76640c7ac89f255b9c41777, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a3713673ae084ff38e51da89eb60f530, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a72e89238ba24306aec3e6b03f796f97, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c56b01d0701342ada1d2a9d03884c7d3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7e9f3c6433d54343aee0e7b87115e02c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/daafeb0a21d5439a95bbf2b613bf7419, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/3700f57bf354479b8bdaa9ac924da087, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e6e9912a27a048078c05d35d919f152b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/0a15f5fb78654926bca7ad36bbfafb5d] to archive 2024-11-24T11:14:46,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:14:46,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7ac6d175ba924a8ea96b5a612494d00b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7ac6d175ba924a8ea96b5a612494d00b 2024-11-24T11:14:46,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/372dd1fcf2c2459d8fc0e24d1d050443 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/372dd1fcf2c2459d8fc0e24d1d050443 2024-11-24T11:14:46,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c299660338e54578bbfcff516f35cf67 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c299660338e54578bbfcff516f35cf67 2024-11-24T11:14:46,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b17d29406eca42189fb162ee631736d8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b17d29406eca42189fb162ee631736d8 2024-11-24T11:14:46,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e73aaf79e354445f8749d519c88a2bdf to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e73aaf79e354445f8749d519c88a2bdf 2024-11-24T11:14:46,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/44ab84aa6e7a4b9db00db03777cdc3ae to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/44ab84aa6e7a4b9db00db03777cdc3ae 2024-11-24T11:14:46,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/876c5e5599934f2ea2b23c080efbf96b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/876c5e5599934f2ea2b23c080efbf96b 2024-11-24T11:14:46,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f771a9565b7d407495045f4f3b6a7b21 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f771a9565b7d407495045f4f3b6a7b21 2024-11-24T11:14:46,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c6b3fbc7ae614f08b7b7c4dbfdbebd77 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c6b3fbc7ae614f08b7b7c4dbfdbebd77 2024-11-24T11:14:46,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/093cb0b03e5e40da98b504ca2403cb46 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/093cb0b03e5e40da98b504ca2403cb46 2024-11-24T11:14:46,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e06308d8bc440b8b142c0d9a7813f8d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e06308d8bc440b8b142c0d9a7813f8d 2024-11-24T11:14:46,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/9830ccc06af34582aefbb0918a5e95d7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/9830ccc06af34582aefbb0918a5e95d7 2024-11-24T11:14:46,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/56f28636aa954733b360f5d0c7118f5f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/56f28636aa954733b360f5d0c7118f5f 2024-11-24T11:14:46,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/2ce7cfadcdda48ba96042c675d49e4b2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/2ce7cfadcdda48ba96042c675d49e4b2 2024-11-24T11:14:46,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/12123cad6e6444eaa7639926b5c18e69 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/12123cad6e6444eaa7639926b5c18e69 2024-11-24T11:14:46,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/5c26bdfbf54142d8ac7fb4b3533f1675 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/5c26bdfbf54142d8ac7fb4b3533f1675 2024-11-24T11:14:46,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/67386cf7ad5d4b23bec9fd0bc5a7001c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/67386cf7ad5d4b23bec9fd0bc5a7001c 2024-11-24T11:14:46,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/cc5c0093a63f478db9a3778d757db110 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/cc5c0093a63f478db9a3778d757db110 2024-11-24T11:14:46,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ab27bb98e19448fcbe9a1ef04427f851 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ab27bb98e19448fcbe9a1ef04427f851 2024-11-24T11:14:46,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f606dd868f7d438dabb601a3232bb75f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f606dd868f7d438dabb601a3232bb75f 2024-11-24T11:14:46,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/91a7e429b9184a00a16fc236433d4474 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/91a7e429b9184a00a16fc236433d4474 2024-11-24T11:14:46,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b74a446187b240f1ae286b0e8f380a1c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b74a446187b240f1ae286b0e8f380a1c 2024-11-24T11:14:46,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bed5e7fc3bb141e69213031f90f3f791 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bed5e7fc3bb141e69213031f90f3f791 2024-11-24T11:14:46,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/241e98089f204143b1f8703bf8034cd4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/241e98089f204143b1f8703bf8034cd4 2024-11-24T11:14:46,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f4b703f4934a42979f21c4a70b2438d2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/f4b703f4934a42979f21c4a70b2438d2 2024-11-24T11:14:46,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bb369bc55bd24473883c579f13e1e143 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/bb369bc55bd24473883c579f13e1e143 2024-11-24T11:14:46,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b9f361549e1f43c58f9ddc50fcc6fcae to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/b9f361549e1f43c58f9ddc50fcc6fcae 2024-11-24T11:14:46,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/04887e38c76640c7ac89f255b9c41777 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/04887e38c76640c7ac89f255b9c41777 2024-11-24T11:14:46,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a3713673ae084ff38e51da89eb60f530 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a3713673ae084ff38e51da89eb60f530 2024-11-24T11:14:46,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a72e89238ba24306aec3e6b03f796f97 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/a72e89238ba24306aec3e6b03f796f97 2024-11-24T11:14:46,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c56b01d0701342ada1d2a9d03884c7d3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/c56b01d0701342ada1d2a9d03884c7d3 2024-11-24T11:14:46,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7e9f3c6433d54343aee0e7b87115e02c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/7e9f3c6433d54343aee0e7b87115e02c 2024-11-24T11:14:46,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/daafeb0a21d5439a95bbf2b613bf7419 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/daafeb0a21d5439a95bbf2b613bf7419 2024-11-24T11:14:46,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/3700f57bf354479b8bdaa9ac924da087 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/3700f57bf354479b8bdaa9ac924da087 2024-11-24T11:14:46,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e6e9912a27a048078c05d35d919f152b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/e6e9912a27a048078c05d35d919f152b 2024-11-24T11:14:46,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/0a15f5fb78654926bca7ad36bbfafb5d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/0a15f5fb78654926bca7ad36bbfafb5d 2024-11-24T11:14:46,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f9d1a309cb644522976a7769bb74ece7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4e59b049915942419961f3db89c6db03, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/5419460d33c54a5587b2d309fe9987d9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/21327fd2d8494860bfd83fa26f1514f2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1271d8b9f1404c56bf85452b028a5579, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/8916584519c140b095d7a6430cd676f2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b95c3c6c2463405fac361a1009ac0bc6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d549924764854cf18d8fabfb33c1b904, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c8ec03ebdd41418d8a7c8718d042eeff, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9dba0fef8e3a486b882015558ac98b64, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/fff760d52fe541e48544393a9f5c7960, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4f6628dff02c468180c7683b341155a2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2f0bea2cb17e4256b00abdfadeee8dcf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/6942bc5d349a42e8927157d5bda5fdb5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2fe75b1d1add46eb8c039e912946c6df, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9b55639b4f694dd28dac45774bb38e5c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/a3816805ea7b4615bc5fa2dfad755695, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/3d29e4b704e240e496c676398696253a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b645edc6f58c4eeab3ba11983d7b2977, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/db3a22e0e3b8435b83b590ada40c582e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e9e280251207425bb68a3018caf15c62, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e3ec0392f74c490099b12738acf67a7d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b205431d239c4f7d8fd3d3fa3dbd181c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1dcb0469467f4fa3834ffb73e899460f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e8d985e3572a4211a261a696dd97b3ce, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/0ecf1c3e2c604e6dbf5a5fb8e71f0c0c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9ffdc6259f4d4254a2a79b427161c6b2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/750c3ef3f5f54b56a2d04700602d0568, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e0262425e8464bc3a9d94f55a4114121, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c3e36931624b47879ec859b5e8b61174, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/782e4562f652421c986536b895c8248b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/36fdbbcce6684c95a98c7303bf67b8d0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d47379304a9a4942b6dd869a44ae79b2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f5f6f78272724ff081f069df8d6aec67, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/284eca873345465faf5c80104d538272, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/102b3b18ea7645cab628d9f3360117c6] to archive 2024-11-24T11:14:46,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:14:46,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f9d1a309cb644522976a7769bb74ece7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f9d1a309cb644522976a7769bb74ece7 2024-11-24T11:14:46,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4e59b049915942419961f3db89c6db03 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4e59b049915942419961f3db89c6db03 2024-11-24T11:14:46,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/5419460d33c54a5587b2d309fe9987d9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/5419460d33c54a5587b2d309fe9987d9 2024-11-24T11:14:46,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/21327fd2d8494860bfd83fa26f1514f2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/21327fd2d8494860bfd83fa26f1514f2 2024-11-24T11:14:46,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1271d8b9f1404c56bf85452b028a5579 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1271d8b9f1404c56bf85452b028a5579 2024-11-24T11:14:46,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/8916584519c140b095d7a6430cd676f2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/8916584519c140b095d7a6430cd676f2 2024-11-24T11:14:46,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b95c3c6c2463405fac361a1009ac0bc6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b95c3c6c2463405fac361a1009ac0bc6 2024-11-24T11:14:46,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d549924764854cf18d8fabfb33c1b904 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d549924764854cf18d8fabfb33c1b904 2024-11-24T11:14:46,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c8ec03ebdd41418d8a7c8718d042eeff to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c8ec03ebdd41418d8a7c8718d042eeff 2024-11-24T11:14:46,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9dba0fef8e3a486b882015558ac98b64 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9dba0fef8e3a486b882015558ac98b64 2024-11-24T11:14:46,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/fff760d52fe541e48544393a9f5c7960 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/fff760d52fe541e48544393a9f5c7960 2024-11-24T11:14:46,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4f6628dff02c468180c7683b341155a2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/4f6628dff02c468180c7683b341155a2 2024-11-24T11:14:46,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2f0bea2cb17e4256b00abdfadeee8dcf to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2f0bea2cb17e4256b00abdfadeee8dcf 2024-11-24T11:14:46,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/6942bc5d349a42e8927157d5bda5fdb5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/6942bc5d349a42e8927157d5bda5fdb5 2024-11-24T11:14:46,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2fe75b1d1add46eb8c039e912946c6df to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/2fe75b1d1add46eb8c039e912946c6df 2024-11-24T11:14:46,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9b55639b4f694dd28dac45774bb38e5c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9b55639b4f694dd28dac45774bb38e5c 2024-11-24T11:14:46,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/a3816805ea7b4615bc5fa2dfad755695 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/a3816805ea7b4615bc5fa2dfad755695 2024-11-24T11:14:46,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/3d29e4b704e240e496c676398696253a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/3d29e4b704e240e496c676398696253a 2024-11-24T11:14:46,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b645edc6f58c4eeab3ba11983d7b2977 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b645edc6f58c4eeab3ba11983d7b2977 2024-11-24T11:14:46,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/db3a22e0e3b8435b83b590ada40c582e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/db3a22e0e3b8435b83b590ada40c582e 2024-11-24T11:14:46,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e9e280251207425bb68a3018caf15c62 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e9e280251207425bb68a3018caf15c62 2024-11-24T11:14:46,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e3ec0392f74c490099b12738acf67a7d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e3ec0392f74c490099b12738acf67a7d 2024-11-24T11:14:46,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b205431d239c4f7d8fd3d3fa3dbd181c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/b205431d239c4f7d8fd3d3fa3dbd181c 2024-11-24T11:14:46,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1dcb0469467f4fa3834ffb73e899460f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1dcb0469467f4fa3834ffb73e899460f 2024-11-24T11:14:46,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e8d985e3572a4211a261a696dd97b3ce to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e8d985e3572a4211a261a696dd97b3ce 2024-11-24T11:14:46,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/0ecf1c3e2c604e6dbf5a5fb8e71f0c0c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/0ecf1c3e2c604e6dbf5a5fb8e71f0c0c 2024-11-24T11:14:46,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9ffdc6259f4d4254a2a79b427161c6b2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/9ffdc6259f4d4254a2a79b427161c6b2 2024-11-24T11:14:46,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/750c3ef3f5f54b56a2d04700602d0568 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/750c3ef3f5f54b56a2d04700602d0568 2024-11-24T11:14:46,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e0262425e8464bc3a9d94f55a4114121 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/e0262425e8464bc3a9d94f55a4114121 2024-11-24T11:14:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T11:14:46,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c3e36931624b47879ec859b5e8b61174 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/c3e36931624b47879ec859b5e8b61174 2024-11-24T11:14:46,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/782e4562f652421c986536b895c8248b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/782e4562f652421c986536b895c8248b 2024-11-24T11:14:46,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/36fdbbcce6684c95a98c7303bf67b8d0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/36fdbbcce6684c95a98c7303bf67b8d0 2024-11-24T11:14:46,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d47379304a9a4942b6dd869a44ae79b2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/d47379304a9a4942b6dd869a44ae79b2 2024-11-24T11:14:46,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f5f6f78272724ff081f069df8d6aec67 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/f5f6f78272724ff081f069df8d6aec67 2024-11-24T11:14:46,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/284eca873345465faf5c80104d538272 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/284eca873345465faf5c80104d538272 2024-11-24T11:14:46,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/102b3b18ea7645cab628d9f3360117c6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/102b3b18ea7645cab628d9f3360117c6 2024-11-24T11:14:46,835 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/recovered.edits/524.seqid, newMaxSeqId=524, maxSeqId=1 2024-11-24T11:14:46,836 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c. 2024-11-24T11:14:46,836 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 358290acb08925e00ab2bb913fcaac3c: Waiting for close lock at 1732446886679Running coprocessor pre-close hooks at 1732446886679Disabling compacts and flushes for region at 1732446886679Disabling writes for close at 1732446886680 (+1 ms)Writing region close event to WAL at 1732446886832 (+152 ms)Running coprocessor post-close hooks at 1732446886835 (+3 ms)Closed at 1732446886835 2024-11-24T11:14:46,838 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:46,838 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=358290acb08925e00ab2bb913fcaac3c, regionState=CLOSED 2024-11-24T11:14:46,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:14:46,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T11:14:46,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 358290acb08925e00ab2bb913fcaac3c, server=f5f73e76a839,45939,1732446856176 in 324 msec 2024-11-24T11:14:46,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T11:14:46,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=358290acb08925e00ab2bb913fcaac3c, UNASSIGN in 330 msec 2024-11-24T11:14:46,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T11:14:46,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 338 msec 2024-11-24T11:14:46,848 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446886847"}]},"ts":"1732446886847"} 2024-11-24T11:14:46,849 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T11:14:46,849 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T11:14:46,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 351 msec 2024-11-24T11:14:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T11:14:47,136 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:47,144 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-24T11:14:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:47,149 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T11:14:47,150 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:47,156 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:47,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/recovered.edits] 2024-11-24T11:14:47,164 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/647ee123c4be4654af148bd8efaf0dcb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/647ee123c4be4654af148bd8efaf0dcb 2024-11-24T11:14:47,165 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/71db110815bd477f98378ef98ba77d97 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/71db110815bd477f98378ef98ba77d97 2024-11-24T11:14:47,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8d58b9eaf0634ee9a2796de97ad6b777 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/A/8d58b9eaf0634ee9a2796de97ad6b777 2024-11-24T11:14:47,169 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/25814845304649b9a9de424773c2bf73 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/25814845304649b9a9de424773c2bf73 2024-11-24T11:14:47,171 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e4a09a24acf4875a345cc7680b48805 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/4e4a09a24acf4875a345cc7680b48805 2024-11-24T11:14:47,172 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ffa7a27a30384b13932febd897ba80fd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/B/ffa7a27a30384b13932febd897ba80fd 2024-11-24T11:14:47,175 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/07900e16472c41fab67260ade5bf26fb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/07900e16472c41fab67260ade5bf26fb 2024-11-24T11:14:47,176 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1cfd3498d2384c4c8ceef7aff0c393cb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/1cfd3498d2384c4c8ceef7aff0c393cb 2024-11-24T11:14:47,178 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/bce52c858416432a8d60aac2d028d199 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/C/bce52c858416432a8d60aac2d028d199 2024-11-24T11:14:47,180 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/recovered.edits/524.seqid to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c/recovered.edits/524.seqid 2024-11-24T11:14:47,181 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/358290acb08925e00ab2bb913fcaac3c 2024-11-24T11:14:47,181 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T11:14:47,186 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:47,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T11:14:47,196 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T11:14:47,231 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T11:14:47,233 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:47,233 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T11:14:47,234 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732446887233"}]},"ts":"9223372036854775807"} 2024-11-24T11:14:47,239 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T11:14:47,239 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 358290acb08925e00ab2bb913fcaac3c, NAME => 'TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T11:14:47,240 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T11:14:47,240 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732446887240"}]},"ts":"9223372036854775807"} 2024-11-24T11:14:47,244 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T11:14:47,245 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:47,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 101 msec 2024-11-24T11:14:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T11:14:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T11:14:47,254 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:47,264 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f5f73e76a839:45939-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1009778533_22 at /127.0.0.1:44764 [Waiting for operation #356] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-452383080_22 at /127.0.0.1:37530 [Waiting for operation #377] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 258) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6236 (was 6812) 2024-11-24T11:14:47,274 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=6235 2024-11-24T11:14:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:14:47,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:14:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:47,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T11:14:47,278 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:47,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-24T11:14:47,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T11:14:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T11:14:47,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741954_1130 (size=1061) 2024-11-24T11:14:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T11:14:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T11:14:47,689 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:14:47,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741955_1131 (size=53) 2024-11-24T11:14:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T11:14:48,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:48,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d57999a7f23de1f386379af6f2171765, disabling compactions & flushes 2024-11-24T11:14:48,103 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. after waiting 0 ms 2024-11-24T11:14:48,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,103 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d57999a7f23de1f386379af6f2171765: Waiting for close lock at 1732446888103Disabling compacts and flushes for region at 1732446888103Disabling writes for close at 1732446888103Writing region close event to WAL at 1732446888103Closed at 1732446888103 2024-11-24T11:14:48,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T11:14:48,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732446888106"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732446888106"}]},"ts":"1732446888106"} 2024-11-24T11:14:48,110 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T11:14:48,112 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T11:14:48,112 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446888112"}]},"ts":"1732446888112"} 2024-11-24T11:14:48,114 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T11:14:48,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, ASSIGN}] 2024-11-24T11:14:48,116 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, ASSIGN 2024-11-24T11:14:48,117 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, ASSIGN; state=OFFLINE, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=false 2024-11-24T11:14:48,268 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:48,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, ASSIGN because future has completed 2024-11-24T11:14:48,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:14:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T11:14:48,437 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,438 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:14:48,438 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,438 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:48,438 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,439 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,440 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,442 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:48,443 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57999a7f23de1f386379af6f2171765 columnFamilyName A 2024-11-24T11:14:48,443 DEBUG [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:48,444 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(327): Store=d57999a7f23de1f386379af6f2171765/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:48,444 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,445 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:48,445 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57999a7f23de1f386379af6f2171765 columnFamilyName B 2024-11-24T11:14:48,446 DEBUG [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:48,446 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(327): Store=d57999a7f23de1f386379af6f2171765/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:48,446 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,447 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:48,447 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57999a7f23de1f386379af6f2171765 columnFamilyName C 2024-11-24T11:14:48,448 DEBUG [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:48,448 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(327): Store=d57999a7f23de1f386379af6f2171765/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:48,448 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,448 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,449 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,449 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,450 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,450 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,451 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:14:48,452 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,454 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:14:48,454 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened d57999a7f23de1f386379af6f2171765; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67592598, jitterRate=0.007208198308944702}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:14:48,454 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:48,455 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for d57999a7f23de1f386379af6f2171765: Running coprocessor pre-open hook at 1732446888439Writing region info on filesystem at 1732446888439Initializing all the Stores at 1732446888440 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446888440Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446888440Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446888440Cleaning up temporary data from old regions at 1732446888450 (+10 ms)Running coprocessor post-open hooks at 1732446888454 (+4 ms)Region opened successfully at 1732446888455 (+1 ms) 2024-11-24T11:14:48,456 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., pid=32, masterSystemTime=1732446888428 2024-11-24T11:14:48,458 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,458 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:48,459 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=OPEN, openSeqNum=2, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:48,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:14:48,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T11:14:48,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 in 187 msec 2024-11-24T11:14:48,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T11:14:48,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, ASSIGN in 350 msec 2024-11-24T11:14:48,466 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T11:14:48,467 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446888466"}]},"ts":"1732446888466"} 2024-11-24T11:14:48,468 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T11:14:48,469 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T11:14:48,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-24T11:14:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T11:14:49,425 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:49,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2bb5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:49,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:49,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:49,432 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39093412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:49,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb8ce1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:49,437 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:49,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:49,441 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T11:14:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f5f73e76a839,40345,1732446855400 2024-11-24T11:14:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3536fd0a 2024-11-24T11:14:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T11:14:49,444 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T11:14:49,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:14:49,449 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:14:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T11:14:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741956_1132 (size=1097) 2024-11-24T11:14:49,871 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T11:14:49,871 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T11:14:49,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:14:49,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, REOPEN/MOVE}] 2024-11-24T11:14:49,886 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, REOPEN/MOVE 2024-11-24T11:14:49,886 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:49,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, REOPEN/MOVE because future has completed 2024-11-24T11:14:49,889 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:14:49,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:14:50,043 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,043 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:14:50,044 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing d57999a7f23de1f386379af6f2171765, disabling compactions & flushes 2024-11-24T11:14:50,044 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,044 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,044 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. after waiting 0 ms 2024-11-24T11:14:50,044 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,051 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T11:14:50,052 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,052 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for d57999a7f23de1f386379af6f2171765: Waiting for close lock at 1732446890043Running coprocessor pre-close hooks at 1732446890043Disabling compacts and flushes for region at 1732446890043Disabling writes for close at 1732446890044 (+1 ms)Writing region close event to WAL at 1732446890045 (+1 ms)Running coprocessor post-close hooks at 1732446890052 (+7 ms)Closed at 1732446890052 2024-11-24T11:14:50,053 WARN [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: d57999a7f23de1f386379af6f2171765 to self. 2024-11-24T11:14:50,056 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,057 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=CLOSED 2024-11-24T11:14:50,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:14:50,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-24T11:14:50,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 in 171 msec 2024-11-24T11:14:50,062 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, REOPEN/MOVE; state=CLOSED, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=true 2024-11-24T11:14:50,213 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, REOPEN/MOVE because future has completed 2024-11-24T11:14:50,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:14:50,659 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,659 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:14:50,659 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,659 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:14:50,659 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,659 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,662 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,662 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:50,666 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57999a7f23de1f386379af6f2171765 columnFamilyName A 2024-11-24T11:14:50,668 DEBUG [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:50,669 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(327): Store=d57999a7f23de1f386379af6f2171765/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:50,669 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,670 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:50,671 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57999a7f23de1f386379af6f2171765 columnFamilyName B 2024-11-24T11:14:50,671 DEBUG [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:50,671 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(327): Store=d57999a7f23de1f386379af6f2171765/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:50,671 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,672 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:14:50,673 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57999a7f23de1f386379af6f2171765 columnFamilyName C 2024-11-24T11:14:50,673 DEBUG [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:50,673 INFO [StoreOpener-d57999a7f23de1f386379af6f2171765-1 {}] regionserver.HStore(327): Store=d57999a7f23de1f386379af6f2171765/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:14:50,674 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,674 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,675 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,676 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,677 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,677 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,678 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:14:50,679 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,681 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened d57999a7f23de1f386379af6f2171765; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73494999, jitterRate=0.0951608270406723}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:14:50,682 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,682 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for d57999a7f23de1f386379af6f2171765: Running coprocessor pre-open hook at 1732446890660Writing region info on filesystem at 1732446890660Initializing all the Stores at 1732446890660Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446890660Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446890661 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446890661Cleaning up temporary data from old regions at 1732446890677 (+16 ms)Running coprocessor post-open hooks at 1732446890682 (+5 ms)Region opened successfully at 1732446890682 2024-11-24T11:14:50,683 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., pid=37, masterSystemTime=1732446890375 2024-11-24T11:14:50,686 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,686 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,686 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=OPEN, openSeqNum=5, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:14:50,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-24T11:14:50,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 in 471 msec 2024-11-24T11:14:50,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T11:14:50,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, REOPEN/MOVE in 807 msec 2024-11-24T11:14:50,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-24T11:14:50,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 818 msec 2024-11-24T11:14:50,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.2450 sec 2024-11-24T11:14:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T11:14:50,702 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:50,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01494b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,704 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4775f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b66406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28944b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,721 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72bea753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,723 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459f9471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95d199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,725 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7a86d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4383f3cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e51468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123b9803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e0f2678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1fcab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,732 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63304016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,733 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551c1ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337f1b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,736 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26437cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,737 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3602d671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62885e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,740 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7665730a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,741 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd015dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138dc9c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,743 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454c435c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,745 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f73a729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c264dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:14:50,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:14:50,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:14:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:14:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:14:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370bb271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:14:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:14:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:14:50,751 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:14:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8b0230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:14:50,752 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,752 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,752 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,753 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,753 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,753 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,754 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,756 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-24T11:14:50,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,759 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:14:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T11:14:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,760 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:14:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:14:50,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,761 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:50,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:14:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:14:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:50,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446950813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446950815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:50,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446950817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:50,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124496e6f260bd94ccda8eed93916dbe54e_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446890773/Put/seqid=0 2024-11-24T11:14:50,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732446890799, max timestamp=1732446890827Num uniques 859; 2024-11-24T11:14:50,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732446890799, max timestamp=1732446890827Num uniques 859; 2024-11-24T11:14:50,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-24T11:14:50,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-24T11:14:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732446890799, max timestamp=1732446890827Num uniques 859; 2024-11-24T11:14:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-24T11:14:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T11:14:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741957_1133 (size=12154) 2024-11-24T11:14:50,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:50,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124496e6f260bd94ccda8eed93916dbe54e_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124496e6f260bd94ccda8eed93916dbe54e_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:50,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/3389713d1de1437dad9dfc3493f8f4a0, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:50,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/3389713d1de1437dad9dfc3493f8f4a0 is 175, key is test_row_0/A:col10/1732446890773/Put/seqid=0 2024-11-24T11:14:50,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741958_1134 (size=30951) 2024-11-24T11:14:50,896 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/3389713d1de1437dad9dfc3493f8f4a0 2024-11-24T11:14:50,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446950890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446950900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:50,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:50,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:50,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:50,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/3bf99ed6a01c4d75ac4c96788d1193c0 is 50, key is test_row_0/B:col10/1732446890773/Put/seqid=0 2024-11-24T11:14:50,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741959_1135 (size=9657) 2024-11-24T11:14:50,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/3bf99ed6a01c4d75ac4c96788d1193c0 2024-11-24T11:14:50,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446950933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446950933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:50,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446950947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/834129da4b204a98a06d6d4d0b8ff3b8 is 50, key is test_row_0/C:col10/1732446890773/Put/seqid=0 2024-11-24T11:14:51,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446951004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446951024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741960_1136 (size=9657) 2024-11-24T11:14:51,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/834129da4b204a98a06d6d4d0b8ff3b8 2024-11-24T11:14:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/3389713d1de1437dad9dfc3493f8f4a0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/3389713d1de1437dad9dfc3493f8f4a0 2024-11-24T11:14:51,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/3389713d1de1437dad9dfc3493f8f4a0, entries=150, sequenceid=15, filesize=30.2 K 2024-11-24T11:14:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/3bf99ed6a01c4d75ac4c96788d1193c0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/3bf99ed6a01c4d75ac4c96788d1193c0 2024-11-24T11:14:51,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/3bf99ed6a01c4d75ac4c96788d1193c0, entries=100, sequenceid=15, filesize=9.4 K 2024-11-24T11:14:51,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/834129da4b204a98a06d6d4d0b8ff3b8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/834129da4b204a98a06d6d4d0b8ff3b8 2024-11-24T11:14:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T11:14:51,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/834129da4b204a98a06d6d4d0b8ff3b8, entries=100, sequenceid=15, filesize=9.4 K 2024-11-24T11:14:51,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d57999a7f23de1f386379af6f2171765 in 309ms, sequenceid=15, compaction requested=false 2024-11-24T11:14:51,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:51,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-24T11:14:51,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:51,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:51,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:51,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:51,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:51,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:51,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244d0b7949ef3b458fa2490208b94f6194_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446890853/Put/seqid=0 2024-11-24T11:14:51,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:51,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:51,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446951212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446951219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446951224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446951234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741961_1137 (size=14594) 2024-11-24T11:14:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446951227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446951354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446951374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446951374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T11:14:51,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446951563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446951574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446951574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446951594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446951594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:51,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244d0b7949ef3b458fa2490208b94f6194_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244d0b7949ef3b458fa2490208b94f6194_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:51,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/2f02d88c6ca2454588b459309c6d05fd, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:51,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/2f02d88c6ca2454588b459309c6d05fd is 175, key is test_row_0/A:col10/1732446890853/Put/seqid=0 2024-11-24T11:14:51,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741962_1138 (size=39549) 2024-11-24T11:14:51,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:51,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:51,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446951884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T11:14:51,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446951903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:51,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446951914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:52,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:52,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/2f02d88c6ca2454588b459309c6d05fd 2024-11-24T11:14:52,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446952094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446952094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/8173bd30bf2f4676a5c2194868c1233b is 50, key is test_row_0/B:col10/1732446890853/Put/seqid=0 2024-11-24T11:14:52,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741963_1139 (size=12001) 2024-11-24T11:14:52,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/8173bd30bf2f4676a5c2194868c1233b 2024-11-24T11:14:52,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/4cbf45dac95943a1a8f6e683709075d0 is 50, key is test_row_0/C:col10/1732446890853/Put/seqid=0 2024-11-24T11:14:52,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741964_1140 (size=12001) 2024-11-24T11:14:52,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/4cbf45dac95943a1a8f6e683709075d0 2024-11-24T11:14:52,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:52,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:52,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:52,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/2f02d88c6ca2454588b459309c6d05fd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/2f02d88c6ca2454588b459309c6d05fd 2024-11-24T11:14:52,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:52,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/2f02d88c6ca2454588b459309c6d05fd, entries=200, sequenceid=48, filesize=38.6 K 2024-11-24T11:14:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/8173bd30bf2f4676a5c2194868c1233b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8173bd30bf2f4676a5c2194868c1233b 2024-11-24T11:14:52,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8173bd30bf2f4676a5c2194868c1233b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T11:14:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/4cbf45dac95943a1a8f6e683709075d0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4cbf45dac95943a1a8f6e683709075d0 2024-11-24T11:14:52,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4cbf45dac95943a1a8f6e683709075d0, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T11:14:52,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for d57999a7f23de1f386379af6f2171765 in 1005ms, sequenceid=48, compaction requested=false 2024-11-24T11:14:52,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:52,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T11:14:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:52,316 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:14:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b5d8a09ab43419f815a3af86c3f6804_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446891204/Put/seqid=0 2024-11-24T11:14:52,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741965_1141 (size=12154) 2024-11-24T11:14:52,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:52,382 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T11:14:52,386 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b5d8a09ab43419f815a3af86c3f6804_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b5d8a09ab43419f815a3af86c3f6804_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:52,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/02447491fa0a460ca686fba32b3eda3d, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/02447491fa0a460ca686fba32b3eda3d is 175, key is test_row_0/A:col10/1732446891204/Put/seqid=0 2024-11-24T11:14:52,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:52,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741966_1142 (size=30955) 2024-11-24T11:14:52,428 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/02447491fa0a460ca686fba32b3eda3d 2024-11-24T11:14:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/a113288301b24d3780ced566fcbc0bb9 is 50, key is test_row_0/B:col10/1732446891204/Put/seqid=0 2024-11-24T11:14:52,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741967_1143 (size=12001) 2024-11-24T11:14:52,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446892451, max timestamp=1732446892469Num uniques 859; 2024-11-24T11:14:52,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:52,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446892451, max timestamp=1732446892469Num uniques 859; 2024-11-24T11:14:52,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446892451, max timestamp=1732446892469Num uniques 859; 2024-11-24T11:14:52,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:52,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:52,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446952509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446952520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446952523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446952623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446952634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446952643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446952834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446952843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446952853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:52,867 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/a113288301b24d3780ced566fcbc0bb9 2024-11-24T11:14:52,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a948109258c042308d14d53b12b0e5fa is 50, key is test_row_0/C:col10/1732446891204/Put/seqid=0 2024-11-24T11:14:52,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741968_1144 (size=12001) 2024-11-24T11:14:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T11:14:53,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446953113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446953113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,123 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,124 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446953143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446953153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446953164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,280 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a948109258c042308d14d53b12b0e5fa 2024-11-24T11:14:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/02447491fa0a460ca686fba32b3eda3d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/02447491fa0a460ca686fba32b3eda3d 2024-11-24T11:14:53,292 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/02447491fa0a460ca686fba32b3eda3d, entries=150, sequenceid=58, filesize=30.2 K 2024-11-24T11:14:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/a113288301b24d3780ced566fcbc0bb9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a113288301b24d3780ced566fcbc0bb9 2024-11-24T11:14:53,304 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a113288301b24d3780ced566fcbc0bb9, entries=150, sequenceid=58, filesize=11.7 K 2024-11-24T11:14:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a948109258c042308d14d53b12b0e5fa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a948109258c042308d14d53b12b0e5fa 2024-11-24T11:14:53,313 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a948109258c042308d14d53b12b0e5fa, entries=150, sequenceid=58, filesize=11.7 K 2024-11-24T11:14:53,314 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for d57999a7f23de1f386379af6f2171765 in 998ms, sequenceid=58, compaction requested=true 2024-11-24T11:14:53,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:53,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:53,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-24T11:14:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-24T11:14:53,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-24T11:14:53,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5550 sec 2024-11-24T11:14:53,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.5650 sec 2024-11-24T11:14:53,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-24T11:14:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T11:14:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T11:14:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T11:14:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:53,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249971adacfb10425ebb18985c543af4d4_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446892507/Put/seqid=0 2024-11-24T11:14:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446953674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446953675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446953679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741969_1145 (size=12154) 2024-11-24T11:14:53,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:53,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249971adacfb10425ebb18985c543af4d4_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249971adacfb10425ebb18985c543af4d4_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:53,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/8a0b10ca8e57439297bb07458a95c59e, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:53,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/8a0b10ca8e57439297bb07458a95c59e is 175, key is test_row_0/A:col10/1732446892507/Put/seqid=0 2024-11-24T11:14:53,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741970_1146 (size=30955) 2024-11-24T11:14:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446953783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446953784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446953794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446954003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446954004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446954004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/8a0b10ca8e57439297bb07458a95c59e 2024-11-24T11:14:54,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/19fd1458448942aab183428d55492932 is 50, key is test_row_0/B:col10/1732446892507/Put/seqid=0 2024-11-24T11:14:54,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741971_1147 (size=12001) 2024-11-24T11:14:54,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/19fd1458448942aab183428d55492932 2024-11-24T11:14:54,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/fa824bb8652b4eeea5cf8096e6b2e1aa is 50, key is test_row_0/C:col10/1732446892507/Put/seqid=0 2024-11-24T11:14:54,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741972_1148 (size=12001) 2024-11-24T11:14:54,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/fa824bb8652b4eeea5cf8096e6b2e1aa 2024-11-24T11:14:54,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/8a0b10ca8e57439297bb07458a95c59e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/8a0b10ca8e57439297bb07458a95c59e 2024-11-24T11:14:54,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/8a0b10ca8e57439297bb07458a95c59e, entries=150, sequenceid=96, filesize=30.2 K 2024-11-24T11:14:54,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/19fd1458448942aab183428d55492932 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/19fd1458448942aab183428d55492932 2024-11-24T11:14:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/19fd1458448942aab183428d55492932, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T11:14:54,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/fa824bb8652b4eeea5cf8096e6b2e1aa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/fa824bb8652b4eeea5cf8096e6b2e1aa 2024-11-24T11:14:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/fa824bb8652b4eeea5cf8096e6b2e1aa, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T11:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for d57999a7f23de1f386379af6f2171765 in 538ms, sequenceid=96, compaction requested=true 2024-11-24T11:14:54,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:54,196 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:54,196 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,198 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:54,199 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/A is initiating minor compaction (all files) 2024-11-24T11:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,199 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/A in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:54,199 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/3389713d1de1437dad9dfc3493f8f4a0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/2f02d88c6ca2454588b459309c6d05fd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/02447491fa0a460ca686fba32b3eda3d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/8a0b10ca8e57439297bb07458a95c59e] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=129.3 K 2024-11-24T11:14:54,199 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:54,199 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/3389713d1de1437dad9dfc3493f8f4a0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/2f02d88c6ca2454588b459309c6d05fd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/02447491fa0a460ca686fba32b3eda3d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/8a0b10ca8e57439297bb07458a95c59e] 2024-11-24T11:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,200 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:54,200 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3389713d1de1437dad9dfc3493f8f4a0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732446890769 2024-11-24T11:14:54,200 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/B is initiating minor compaction (all files) 2024-11-24T11:14:54,200 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/B in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:54,200 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/3bf99ed6a01c4d75ac4c96788d1193c0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8173bd30bf2f4676a5c2194868c1233b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a113288301b24d3780ced566fcbc0bb9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/19fd1458448942aab183428d55492932] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=44.6 K 2024-11-24T11:14:54,200 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f02d88c6ca2454588b459309c6d05fd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732446890853 2024-11-24T11:14:54,201 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf99ed6a01c4d75ac4c96788d1193c0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732446890769 2024-11-24T11:14:54,201 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02447491fa0a460ca686fba32b3eda3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732446891204 2024-11-24T11:14:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,201 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8173bd30bf2f4676a5c2194868c1233b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732446890853 2024-11-24T11:14:54,201 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0b10ca8e57439297bb07458a95c59e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446892491 2024-11-24T11:14:54,202 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a113288301b24d3780ced566fcbc0bb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732446891204 2024-11-24T11:14:54,202 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 19fd1458448942aab183428d55492932, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446892491 2024-11-24T11:14:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,223 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:54,224 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:54,224 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/9357d73cda044dd4b1293369bb997b74 is 50, key is test_row_0/B:col10/1732446892507/Put/seqid=0 2024-11-24T11:14:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,227 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411243e4549ffe1a74c31bec844104e9713ae_d57999a7f23de1f386379af6f2171765 store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,234 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411243e4549ffe1a74c31bec844104e9713ae_d57999a7f23de1f386379af6f2171765, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:54,234 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243e4549ffe1a74c31bec844104e9713ae_d57999a7f23de1f386379af6f2171765 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741973_1149 (size=12139) 2024-11-24T11:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,258 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/9357d73cda044dd4b1293369bb997b74 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/9357d73cda044dd4b1293369bb997b74 2024-11-24T11:14:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,269 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/B of d57999a7f23de1f386379af6f2171765 into 9357d73cda044dd4b1293369bb997b74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:54,269 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:54,269 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/B, priority=12, startTime=1732446894196; duration=0sec 2024-11-24T11:14:54,269 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:54,269 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:B 2024-11-24T11:14:54,269 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:54,271 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:54,271 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/C is initiating minor compaction (all files) 2024-11-24T11:14:54,271 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/C in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:54,271 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/834129da4b204a98a06d6d4d0b8ff3b8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4cbf45dac95943a1a8f6e683709075d0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a948109258c042308d14d53b12b0e5fa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/fa824bb8652b4eeea5cf8096e6b2e1aa] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=44.6 K 2024-11-24T11:14:54,272 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 834129da4b204a98a06d6d4d0b8ff3b8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732446890769 2024-11-24T11:14:54,272 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cbf45dac95943a1a8f6e683709075d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732446890853 2024-11-24T11:14:54,273 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a948109258c042308d14d53b12b0e5fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732446891204 2024-11-24T11:14:54,273 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting fa824bb8652b4eeea5cf8096e6b2e1aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446892491 2024-11-24T11:14:54,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741974_1150 (size=4469) 2024-11-24T11:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,286 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#A#compaction#130 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,287 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/c5253ba5a4ea470aa99e9e470254d1a5 is 175, key is test_row_0/A:col10/1732446892507/Put/seqid=0 2024-11-24T11:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,296 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#C#compaction#131 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,297 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/1796f18bb0d543e69f98945bd9a857ce is 50, key is test_row_0/C:col10/1732446892507/Put/seqid=0 2024-11-24T11:14:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741975_1151 (size=31093) 2024-11-24T11:14:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,322 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/c5253ba5a4ea470aa99e9e470254d1a5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c5253ba5a4ea470aa99e9e470254d1a5 2024-11-24T11:14:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,329 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/A of d57999a7f23de1f386379af6f2171765 into c5253ba5a4ea470aa99e9e470254d1a5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:54,329 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:54,329 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/A, priority=12, startTime=1732446894195; duration=0sec 2024-11-24T11:14:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,329 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:54,329 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:A 2024-11-24T11:14:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741976_1152 (size=12139) 2024-11-24T11:14:54,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:14:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124905e4e1d8eaf4146aebb0dd030589108_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446893673/Put/seqid=0 2024-11-24T11:14:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741977_1153 (size=14594) 2024-11-24T11:14:54,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446894367, max timestamp=1732446894383Num uniques 859; 2024-11-24T11:14:54,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446894367, max timestamp=1732446894383Num uniques 859; 2024-11-24T11:14:54,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:14:54,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:14:54,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446954414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446954416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446954417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446954534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446954535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446954535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,740 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/1796f18bb0d543e69f98945bd9a857ce as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/1796f18bb0d543e69f98945bd9a857ce 2024-11-24T11:14:54,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446954744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446954744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446954744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:54,759 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/C of d57999a7f23de1f386379af6f2171765 into 1796f18bb0d543e69f98945bd9a857ce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:54,759 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:54,760 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/C, priority=12, startTime=1732446894196; duration=0sec 2024-11-24T11:14:54,760 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:54,760 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:C 2024-11-24T11:14:54,767 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:54,774 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124905e4e1d8eaf4146aebb0dd030589108_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124905e4e1d8eaf4146aebb0dd030589108_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:54,776 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/67143247cdf34bd692937efe87c2e669, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:54,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/67143247cdf34bd692937efe87c2e669 is 175, key is test_row_0/A:col10/1732446893673/Put/seqid=0 2024-11-24T11:14:54,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741978_1154 (size=39549) 2024-11-24T11:14:54,794 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/67143247cdf34bd692937efe87c2e669 2024-11-24T11:14:54,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/4bb638dc49c44665abc729c4b0201c56 is 50, key is test_row_0/B:col10/1732446893673/Put/seqid=0 2024-11-24T11:14:54,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741979_1155 (size=12001) 2024-11-24T11:14:54,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/4bb638dc49c44665abc729c4b0201c56 2024-11-24T11:14:54,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/ae4f84a923d34debbcff7cd996723c38 is 50, key is test_row_0/C:col10/1732446893673/Put/seqid=0 2024-11-24T11:14:54,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741980_1156 (size=12001) 2024-11-24T11:14:54,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/ae4f84a923d34debbcff7cd996723c38 2024-11-24T11:14:54,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/67143247cdf34bd692937efe87c2e669 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/67143247cdf34bd692937efe87c2e669 2024-11-24T11:14:54,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/67143247cdf34bd692937efe87c2e669, entries=200, sequenceid=108, filesize=38.6 K 2024-11-24T11:14:54,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/4bb638dc49c44665abc729c4b0201c56 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/4bb638dc49c44665abc729c4b0201c56 2024-11-24T11:14:54,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/4bb638dc49c44665abc729c4b0201c56, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T11:14:54,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/ae4f84a923d34debbcff7cd996723c38 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/ae4f84a923d34debbcff7cd996723c38 2024-11-24T11:14:54,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/ae4f84a923d34debbcff7cd996723c38, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T11:14:54,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d57999a7f23de1f386379af6f2171765 in 548ms, sequenceid=108, compaction requested=false 2024-11-24T11:14:54,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T11:14:54,934 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:54,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-24T11:14:54,938 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T11:14:54,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:54,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T11:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:55,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=140.89 KB heapSize=403.63 KB 2024-11-24T11:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:55,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246422b89bfaa04049a217b577d19765ae_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446895059/Put/seqid=0 2024-11-24T11:14:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446955068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446955072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446955072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741981_1157 (size=17284) 2024-11-24T11:14:55,083 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246422b89bfaa04049a217b577d19765ae_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246422b89bfaa04049a217b577d19765ae_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:55,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/c58727ef24f14dbabfca6bf080a884db, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:55,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/c58727ef24f14dbabfca6bf080a884db is 175, key is test_row_0/A:col10/1732446895059/Put/seqid=0 2024-11-24T11:14:55,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T11:14:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741982_1158 (size=48389) 2024-11-24T11:14:55,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/c58727ef24f14dbabfca6bf080a884db 2024-11-24T11:14:55,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/cc69b13bd45243758b869c33c12cf9ef is 50, key is test_row_0/B:col10/1732446895059/Put/seqid=0 2024-11-24T11:14:55,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741983_1159 (size=12151) 2024-11-24T11:14:55,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/cc69b13bd45243758b869c33c12cf9ef 2024-11-24T11:14:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446955134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,138 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/8613bede92344c49b19c6c081883ca9d is 50, key is test_row_0/C:col10/1732446895059/Put/seqid=0 2024-11-24T11:14:55,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446955144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,148 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741984_1160 (size=12151) 2024-11-24T11:14:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446955183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446955183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446955184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T11:14:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T11:14:55,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446955394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446955394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446955394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:55,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T11:14:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T11:14:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:55,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/8613bede92344c49b19c6c081883ca9d 2024-11-24T11:14:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T11:14:55,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/c58727ef24f14dbabfca6bf080a884db as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c58727ef24f14dbabfca6bf080a884db 2024-11-24T11:14:55,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c58727ef24f14dbabfca6bf080a884db, entries=250, sequenceid=142, filesize=47.3 K 2024-11-24T11:14:55,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/cc69b13bd45243758b869c33c12cf9ef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/cc69b13bd45243758b869c33c12cf9ef 2024-11-24T11:14:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/cc69b13bd45243758b869c33c12cf9ef, entries=150, sequenceid=142, filesize=11.9 K 2024-11-24T11:14:55,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/8613bede92344c49b19c6c081883ca9d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/8613bede92344c49b19c6c081883ca9d 2024-11-24T11:14:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/8613bede92344c49b19c6c081883ca9d, entries=150, sequenceid=142, filesize=11.9 K 2024-11-24T11:14:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~423.50 KB/433664, currentSize=33.54 KB/34350 for d57999a7f23de1f386379af6f2171765 in 527ms, sequenceid=142, compaction requested=true 2024-11-24T11:14:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:55,588 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:55,588 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,589 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:55,589 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:55,589 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/A is initiating minor compaction (all files) 2024-11-24T11:14:55,589 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/B is initiating minor compaction (all files) 2024-11-24T11:14:55,590 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/A in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,590 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/B in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,590 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/9357d73cda044dd4b1293369bb997b74, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/4bb638dc49c44665abc729c4b0201c56, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/cc69b13bd45243758b869c33c12cf9ef] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=35.4 K 2024-11-24T11:14:55,590 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c5253ba5a4ea470aa99e9e470254d1a5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/67143247cdf34bd692937efe87c2e669, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c58727ef24f14dbabfca6bf080a884db] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=116.2 K 2024-11-24T11:14:55,590 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,590 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c5253ba5a4ea470aa99e9e470254d1a5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/67143247cdf34bd692937efe87c2e669, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c58727ef24f14dbabfca6bf080a884db] 2024-11-24T11:14:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,590 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9357d73cda044dd4b1293369bb997b74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446892491 2024-11-24T11:14:55,590 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5253ba5a4ea470aa99e9e470254d1a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446892491 2024-11-24T11:14:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,591 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb638dc49c44665abc729c4b0201c56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732446893673 2024-11-24T11:14:55,591 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67143247cdf34bd692937efe87c2e669, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732446893673 2024-11-24T11:14:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,592 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting cc69b13bd45243758b869c33c12cf9ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732446894403 2024-11-24T11:14:55,592 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting c58727ef24f14dbabfca6bf080a884db, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732446894403 2024-11-24T11:14:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,605 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#B#compaction#138 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:55,605 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/85e5beec56a74115b79d87280cb6a92e is 50, key is test_row_0/B:col10/1732446895059/Put/seqid=0 2024-11-24T11:14:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,608 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,625 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112417215c65c4d14758847627563f47c269_d57999a7f23de1f386379af6f2171765 store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,630 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112417215c65c4d14758847627563f47c269_d57999a7f23de1f386379af6f2171765, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,630 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112417215c65c4d14758847627563f47c269_d57999a7f23de1f386379af6f2171765 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741986_1162 (size=4469) 2024-11-24T11:14:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,644 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#A#compaction#139 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:55,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741985_1161 (size=12391) 2024-11-24T11:14:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,645 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/e52c906e30d842b981f223ee7b2cd99b is 175, key is test_row_0/A:col10/1732446895059/Put/seqid=0 2024-11-24T11:14:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741987_1163 (size=31345) 2024-11-24T11:14:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,651 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/85e5beec56a74115b79d87280cb6a92e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/85e5beec56a74115b79d87280cb6a92e 2024-11-24T11:14:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,657 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/B of d57999a7f23de1f386379af6f2171765 into 85e5beec56a74115b79d87280cb6a92e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:55,657 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:55,657 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/B, priority=13, startTime=1732446895588; duration=0sec 2024-11-24T11:14:55,657 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:55,657 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:B 2024-11-24T11:14:55,657 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:14:55,659 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:14:55,659 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/C is initiating minor compaction (all files) 2024-11-24T11:14:55,659 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/C in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,659 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/1796f18bb0d543e69f98945bd9a857ce, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/ae4f84a923d34debbcff7cd996723c38, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/8613bede92344c49b19c6c081883ca9d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=35.4 K 2024-11-24T11:14:55,660 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1796f18bb0d543e69f98945bd9a857ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446892491 2024-11-24T11:14:55,660 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4f84a923d34debbcff7cd996723c38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732446893673 2024-11-24T11:14:55,660 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8613bede92344c49b19c6c081883ca9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732446894403 2024-11-24T11:14:55,671 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#C#compaction#140 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:55,671 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a216ad8e53fb4c56ae57f46c18bb128a is 50, key is test_row_0/C:col10/1732446895059/Put/seqid=0 2024-11-24T11:14:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741988_1164 (size=12391) 2024-11-24T11:14:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T11:14:55,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:55,708 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T11:14:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:55,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:55,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:55,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:55,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:55,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b1a3b18428cb44bfabde81037119f5dc_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446895069/Put/seqid=0 2024-11-24T11:14:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,738 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] io.ByteBuffAllocator(384): Pool already reached its max capacity : 186 and no free buffers now. Consider increasing the value for 'hbase.server.allocator.max.buffer.count' ? 2024-11-24T11:14:55,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] io.ByteBuffAllocator(384): Pool already reached its max capacity : 186 and no free buffers now. Consider increasing the value for 'hbase.server.allocator.max.buffer.count' ? 2024-11-24T11:14:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741989_1165 (size=7324) 2024-11-24T11:14:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:55,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:55,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446895824, max timestamp=1732446895853Num uniques 859; 2024-11-24T11:14:55,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446895824, max timestamp=1732446895852Num uniques 859; 2024-11-24T11:14:55,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:55,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:55,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732446895824, max timestamp=1732446895852Num uniques 859; 2024-11-24T11:14:55,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:14:55,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732446895880, max timestamp=1732446895907Num uniques 1068; 2024-11-24T11:14:55,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732446895880, max timestamp=1732446895907Num uniques 1068; 2024-11-24T11:14:55,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-24T11:14:55,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-24T11:14:55,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732446895880, max timestamp=1732446895907Num uniques 1068; 2024-11-24T11:14:55,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-24T11:14:56,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446955994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732446895973, max timestamp=1732446895999Num uniques 1177; 2024-11-24T11:14:56,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732446895973, max timestamp=1732446895999Num uniques 1177; 2024-11-24T11:14:56,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-24T11:14:56,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-24T11:14:56,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732446895973, max timestamp=1732446895999Num uniques 1177; 2024-11-24T11:14:56,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-24T11:14:56,061 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/e52c906e30d842b981f223ee7b2cd99b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/e52c906e30d842b981f223ee7b2cd99b 2024-11-24T11:14:56,071 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/A of d57999a7f23de1f386379af6f2171765 into e52c906e30d842b981f223ee7b2cd99b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:56,071 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:56,071 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/A, priority=13, startTime=1732446895588; duration=0sec 2024-11-24T11:14:56,071 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:56,071 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:A 2024-11-24T11:14:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T11:14:56,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732446896043, max timestamp=1732446896071Num uniques 1086; 2024-11-24T11:14:56,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-24T11:14:56,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732446896043, max timestamp=1732446896071Num uniques 1086; 2024-11-24T11:14:56,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-24T11:14:56,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732446896043, max timestamp=1732446896071Num uniques 1036; 2024-11-24T11:14:56,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-24T11:14:56,090 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a216ad8e53fb4c56ae57f46c18bb128a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a216ad8e53fb4c56ae57f46c18bb128a 2024-11-24T11:14:56,101 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/C of d57999a7f23de1f386379af6f2171765 into a216ad8e53fb4c56ae57f46c18bb128a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:56,101 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:56,101 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/C, priority=13, startTime=1732446895588; duration=0sec 2024-11-24T11:14:56,101 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:56,102 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:C 2024-11-24T11:14:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446956132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732446896106, max timestamp=1732446896136Num uniques 1095; 2024-11-24T11:14:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T11:14:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732446896106, max timestamp=1732446896136Num uniques 1095; 2024-11-24T11:14:56,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T11:14:56,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732446896106, max timestamp=1732446896136Num uniques 1095; 2024-11-24T11:14:56,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T11:14:56,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:56,160 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b1a3b18428cb44bfabde81037119f5dc_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b1a3b18428cb44bfabde81037119f5dc_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d5da3d59781445fcbc7837a5108a9405, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:56,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d5da3d59781445fcbc7837a5108a9405 is 175, key is test_row_0/A:col10/1732446895069/Put/seqid=0 2024-11-24T11:14:56,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741990_1166 (size=13815) 2024-11-24T11:14:56,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446956190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446956191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446956244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446956303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446956303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446956454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446956513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446956513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,574 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d5da3d59781445fcbc7837a5108a9405 2024-11-24T11:14:56,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/381ee5163aff467b9f33c868b2901849 is 50, key is test_row_0/B:col10/1732446895069/Put/seqid=0 2024-11-24T11:14:56,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741991_1167 (size=7365) 2024-11-24T11:14:56,588 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/381ee5163aff467b9f33c868b2901849 2024-11-24T11:14:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/6ec2888873e9449484f270492b8ec24b is 50, key is test_row_0/C:col10/1732446895069/Put/seqid=0 2024-11-24T11:14:56,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741992_1168 (size=7365) 2024-11-24T11:14:56,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446956763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446956823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446956823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:57,005 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/6ec2888873e9449484f270492b8ec24b 2024-11-24T11:14:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d5da3d59781445fcbc7837a5108a9405 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d5da3d59781445fcbc7837a5108a9405 2024-11-24T11:14:57,017 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d5da3d59781445fcbc7837a5108a9405, entries=50, sequenceid=151, filesize=13.5 K 2024-11-24T11:14:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/381ee5163aff467b9f33c868b2901849 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/381ee5163aff467b9f33c868b2901849 2024-11-24T11:14:57,025 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/381ee5163aff467b9f33c868b2901849, entries=50, sequenceid=151, filesize=7.2 K 2024-11-24T11:14:57,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/6ec2888873e9449484f270492b8ec24b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/6ec2888873e9449484f270492b8ec24b 2024-11-24T11:14:57,036 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/6ec2888873e9449484f270492b8ec24b, entries=50, sequenceid=151, filesize=7.2 K 2024-11-24T11:14:57,037 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for d57999a7f23de1f386379af6f2171765 in 1329ms, sequenceid=151, compaction requested=false 2024-11-24T11:14:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-24T11:14:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-24T11:14:57,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-24T11:14:57,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1000 sec 2024-11-24T11:14:57,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.1120 sec 2024-11-24T11:14:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T11:14:57,084 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:57,084 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-24T11:14:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T11:14:57,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:57,087 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:57,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T11:14:57,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T11:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:57,240 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=147.60 KB heapSize=412.08 KB 2024-11-24T11:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:57,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248ef8cb7e115f45d9bd4f5f195eaff2e8_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446896187/Put/seqid=0 2024-11-24T11:14:57,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741993_1169 (size=12354) 2024-11-24T11:14:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:57,272 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248ef8cb7e115f45d9bd4f5f195eaff2e8_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ef8cb7e115f45d9bd4f5f195eaff2e8_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:57,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d124c15f7beb4822b5176c58261f6572, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:57,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d124c15f7beb4822b5176c58261f6572 is 175, key is test_row_0/A:col10/1732446896187/Put/seqid=0 2024-11-24T11:14:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:57,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741994_1170 (size=31155) 2024-11-24T11:14:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446957319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:57,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446957334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446957334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T11:14:57,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446957433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446957644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:57,705 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d124c15f7beb4822b5176c58261f6572 2024-11-24T11:14:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T11:14:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/33a6a9573a934e4eb02ad3a410abb0b0 is 50, key is test_row_0/B:col10/1732446896187/Put/seqid=0 2024-11-24T11:14:57,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741995_1171 (size=12201) 2024-11-24T11:14:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446957953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,118 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/33a6a9573a934e4eb02ad3a410abb0b0 2024-11-24T11:14:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/5aade8836c7141e28cd1401470572eff is 50, key is test_row_0/C:col10/1732446896187/Put/seqid=0 2024-11-24T11:14:58,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741996_1172 (size=12201) 2024-11-24T11:14:58,131 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/5aade8836c7141e28cd1401470572eff 2024-11-24T11:14:58,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/d124c15f7beb4822b5176c58261f6572 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d124c15f7beb4822b5176c58261f6572 2024-11-24T11:14:58,141 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d124c15f7beb4822b5176c58261f6572, entries=150, sequenceid=258, filesize=30.4 K 2024-11-24T11:14:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/33a6a9573a934e4eb02ad3a410abb0b0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/33a6a9573a934e4eb02ad3a410abb0b0 2024-11-24T11:14:58,147 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/33a6a9573a934e4eb02ad3a410abb0b0, entries=150, sequenceid=258, filesize=11.9 K 2024-11-24T11:14:58,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/5aade8836c7141e28cd1401470572eff as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/5aade8836c7141e28cd1401470572eff 2024-11-24T11:14:58,153 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/5aade8836c7141e28cd1401470572eff, entries=150, sequenceid=258, filesize=11.9 K 2024-11-24T11:14:58,154 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~412.03 KB/421920, currentSize=33.54 KB/34350 for d57999a7f23de1f386379af6f2171765 in 914ms, sequenceid=258, compaction requested=true 2024-11-24T11:14:58,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:58,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-24T11:14:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-24T11:14:58,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T11:14:58,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-24T11:14:58,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0750 sec 2024-11-24T11:14:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T11:14:58,223 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:14:58,224 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:14:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-24T11:14:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T11:14:58,225 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:14:58,226 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:14:58,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:14:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T11:14:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:58,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:14:58,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:58,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:58,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:58,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:58,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:58,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:14:58,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112407698ce20c524fbf8e139b5fbd3766d7_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446898354/Put/seqid=0 2024-11-24T11:14:58,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T11:14:58,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:58,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741997_1173 (size=14994) 2024-11-24T11:14:58,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:58,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112407698ce20c524fbf8e139b5fbd3766d7_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112407698ce20c524fbf8e139b5fbd3766d7_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:58,388 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/07d251fa6f2e4c0d88f94f0462455468, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:58,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/07d251fa6f2e4c0d88f94f0462455468 is 175, key is test_row_0/A:col10/1732446898354/Put/seqid=0 2024-11-24T11:14:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741998_1174 (size=39949) 2024-11-24T11:14:58,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446898394, max timestamp=1732446898409Num uniques 859; 2024-11-24T11:14:58,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446898394, max timestamp=1732446898409Num uniques 859; 2024-11-24T11:14:58,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446898394, max timestamp=1732446898409Num uniques 859; 2024-11-24T11:14:58,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:14:58,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:14:58,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:14:58,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446958462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446958463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446958470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T11:14:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T11:14:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446958574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446958574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T11:14:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446958783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:58,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446958783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:58,793 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/07d251fa6f2e4c0d88f94f0462455468 2024-11-24T11:14:58,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/26b9f8314247476db930d7822e9246ef is 50, key is test_row_0/B:col10/1732446898354/Put/seqid=0 2024-11-24T11:14:58,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741999_1175 (size=12301) 2024-11-24T11:14:58,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T11:14:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T11:14:58,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T11:14:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:58,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446959093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446959093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T11:14:59,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:59,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:59,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:59,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:59,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:14:59,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446959153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,155 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446959164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,173 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:14:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/26b9f8314247476db930d7822e9246ef 2024-11-24T11:14:59,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/31c24053378e4df995b51f03dbee4e92 is 50, key is test_row_0/C:col10/1732446898354/Put/seqid=0 2024-11-24T11:14:59,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742000_1176 (size=12301) 2024-11-24T11:14:59,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/31c24053378e4df995b51f03dbee4e92 2024-11-24T11:14:59,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/07d251fa6f2e4c0d88f94f0462455468 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/07d251fa6f2e4c0d88f94f0462455468 2024-11-24T11:14:59,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/07d251fa6f2e4c0d88f94f0462455468, entries=200, sequenceid=268, filesize=39.0 K 2024-11-24T11:14:59,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/26b9f8314247476db930d7822e9246ef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/26b9f8314247476db930d7822e9246ef 2024-11-24T11:14:59,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/26b9f8314247476db930d7822e9246ef, entries=150, sequenceid=268, filesize=12.0 K 2024-11-24T11:14:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/31c24053378e4df995b51f03dbee4e92 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/31c24053378e4df995b51f03dbee4e92 2024-11-24T11:14:59,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/31c24053378e4df995b51f03dbee4e92, entries=150, sequenceid=268, filesize=12.0 K 2024-11-24T11:14:59,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d57999a7f23de1f386379af6f2171765 in 906ms, sequenceid=268, compaction requested=true 2024-11-24T11:14:59,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:14:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:59,262 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:14:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:59,262 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:14:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:59,263 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:59,263 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 116264 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:59,263 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/A is initiating minor compaction (all files) 2024-11-24T11:14:59,263 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/B is initiating minor compaction (all files) 2024-11-24T11:14:59,263 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/B in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:59,263 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/A in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:59,263 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/e52c906e30d842b981f223ee7b2cd99b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d5da3d59781445fcbc7837a5108a9405, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d124c15f7beb4822b5176c58261f6572, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/07d251fa6f2e4c0d88f94f0462455468] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=113.5 K 2024-11-24T11:14:59,263 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/85e5beec56a74115b79d87280cb6a92e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/381ee5163aff467b9f33c868b2901849, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/33a6a9573a934e4eb02ad3a410abb0b0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/26b9f8314247476db930d7822e9246ef] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=43.2 K 2024-11-24T11:14:59,263 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:59,264 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/e52c906e30d842b981f223ee7b2cd99b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d5da3d59781445fcbc7837a5108a9405, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d124c15f7beb4822b5176c58261f6572, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/07d251fa6f2e4c0d88f94f0462455468] 2024-11-24T11:14:59,264 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52c906e30d842b981f223ee7b2cd99b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732446894403 2024-11-24T11:14:59,264 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e5beec56a74115b79d87280cb6a92e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732446894403 2024-11-24T11:14:59,264 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5da3d59781445fcbc7837a5108a9405, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732446895069 2024-11-24T11:14:59,264 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 381ee5163aff467b9f33c868b2901849, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732446895069 2024-11-24T11:14:59,265 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d124c15f7beb4822b5176c58261f6572, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732446896187 2024-11-24T11:14:59,265 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a6a9573a934e4eb02ad3a410abb0b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732446896187 2024-11-24T11:14:59,265 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d251fa6f2e4c0d88f94f0462455468, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732446897310 2024-11-24T11:14:59,265 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b9f8314247476db930d7822e9246ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732446897317 2024-11-24T11:14:59,287 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:59,288 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/e63c6b4da78d41c2922a96f3c181a7f0 is 50, key is test_row_0/B:col10/1732446898354/Put/seqid=0 2024-11-24T11:14:59,291 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:59,294 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411249efa8e04cd8b49e7b6ee8c25ab8e00d0_d57999a7f23de1f386379af6f2171765 store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:59,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T11:14:59,297 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411249efa8e04cd8b49e7b6ee8c25ab8e00d0_d57999a7f23de1f386379af6f2171765, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:59,297 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249efa8e04cd8b49e7b6ee8c25ab8e00d0_d57999a7f23de1f386379af6f2171765 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:59,297 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T11:14:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:14:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:59,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:14:59,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:59,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:14:59,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:14:59,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742001_1177 (size=12677) 2024-11-24T11:14:59,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742002_1178 (size=4469) 2024-11-24T11:14:59,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242838a0d2f7c64febbfd7a310b31e2211_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446898459/Put/seqid=0 2024-11-24T11:14:59,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742003_1179 (size=12454) 2024-11-24T11:14:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T11:14:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:59,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:14:59,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446959514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446959603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446959614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446959624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:14:59,713 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/e63c6b4da78d41c2922a96f3c181a7f0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/e63c6b4da78d41c2922a96f3c181a7f0 2024-11-24T11:14:59,715 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#A#compaction#151 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:59,716 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/4c733a14bb9b4fdd964e24ae42ff413c is 175, key is test_row_0/A:col10/1732446898354/Put/seqid=0 2024-11-24T11:14:59,720 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/B of d57999a7f23de1f386379af6f2171765 into e63c6b4da78d41c2922a96f3c181a7f0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:59,720 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:59,720 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/B, priority=12, startTime=1732446899262; duration=0sec 2024-11-24T11:14:59,720 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:14:59,721 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:B 2024-11-24T11:14:59,721 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:14:59,722 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:14:59,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742004_1180 (size=31631) 2024-11-24T11:14:59,723 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/C is initiating minor compaction (all files) 2024-11-24T11:14:59,723 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/C in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:14:59,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:14:59,724 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a216ad8e53fb4c56ae57f46c18bb128a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/6ec2888873e9449484f270492b8ec24b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/5aade8836c7141e28cd1401470572eff, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/31c24053378e4df995b51f03dbee4e92] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=43.2 K 2024-11-24T11:14:59,725 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a216ad8e53fb4c56ae57f46c18bb128a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732446894403 2024-11-24T11:14:59,725 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ec2888873e9449484f270492b8ec24b, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732446895069 2024-11-24T11:14:59,726 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aade8836c7141e28cd1401470572eff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732446896187 2024-11-24T11:14:59,726 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 31c24053378e4df995b51f03dbee4e92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732446897317 2024-11-24T11:14:59,728 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242838a0d2f7c64febbfd7a310b31e2211_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242838a0d2f7c64febbfd7a310b31e2211_d57999a7f23de1f386379af6f2171765 2024-11-24T11:14:59,729 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/4c733a14bb9b4fdd964e24ae42ff413c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/4c733a14bb9b4fdd964e24ae42ff413c 2024-11-24T11:14:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/9e60ec57d19045fe9fc794a399e4b2b0, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:14:59,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/9e60ec57d19045fe9fc794a399e4b2b0 is 175, key is test_row_0/A:col10/1732446898459/Put/seqid=0 2024-11-24T11:14:59,736 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/A of d57999a7f23de1f386379af6f2171765 into 4c733a14bb9b4fdd964e24ae42ff413c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:14:59,736 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:14:59,736 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/A, priority=12, startTime=1732446899261; duration=0sec 2024-11-24T11:14:59,736 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:14:59,736 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:A 2024-11-24T11:14:59,743 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:14:59,744 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/7d92c0dbb0c0496f9228f73d29f962c0 is 50, key is test_row_0/C:col10/1732446898354/Put/seqid=0 2024-11-24T11:14:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742005_1181 (size=31255) 2024-11-24T11:14:59,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742006_1182 (size=12677) 2024-11-24T11:14:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:14:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446959833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:14:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446960144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:00,150 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/9e60ec57d19045fe9fc794a399e4b2b0 2024-11-24T11:15:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/ed45e2a576a74ab6b2d033bfd96d83e6 is 50, key is test_row_0/B:col10/1732446898459/Put/seqid=0 2024-11-24T11:15:00,164 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/7d92c0dbb0c0496f9228f73d29f962c0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d92c0dbb0c0496f9228f73d29f962c0 2024-11-24T11:15:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742007_1183 (size=12301) 2024-11-24T11:15:00,170 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/C of d57999a7f23de1f386379af6f2171765 into 7d92c0dbb0c0496f9228f73d29f962c0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:00,170 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:00,170 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/C, priority=12, startTime=1732446899262; duration=0sec 2024-11-24T11:15:00,170 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:00,170 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:C 2024-11-24T11:15:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T11:15:00,565 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/ed45e2a576a74ab6b2d033bfd96d83e6 2024-11-24T11:15:00,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446960614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:00,615 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2145 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/e94d4dcad50e4a4e946ea2019f338cce is 50, key is test_row_0/C:col10/1732446898459/Put/seqid=0 2024-11-24T11:15:00,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742008_1184 (size=12301) 2024-11-24T11:15:00,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446960633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:00,635 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:00,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446960653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:00,665 INFO [master/f5f73e76a839:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T11:15:00,665 INFO [master/f5f73e76a839:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T11:15:01,027 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/e94d4dcad50e4a4e946ea2019f338cce 2024-11-24T11:15:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/9e60ec57d19045fe9fc794a399e4b2b0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9e60ec57d19045fe9fc794a399e4b2b0 2024-11-24T11:15:01,035 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9e60ec57d19045fe9fc794a399e4b2b0, entries=150, sequenceid=303, filesize=30.5 K 2024-11-24T11:15:01,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/ed45e2a576a74ab6b2d033bfd96d83e6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ed45e2a576a74ab6b2d033bfd96d83e6 2024-11-24T11:15:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,041 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ed45e2a576a74ab6b2d033bfd96d83e6, entries=150, sequenceid=303, filesize=12.0 K 2024-11-24T11:15:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/e94d4dcad50e4a4e946ea2019f338cce as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/e94d4dcad50e4a4e946ea2019f338cce 2024-11-24T11:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,046 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/e94d4dcad50e4a4e946ea2019f338cce, entries=150, sequenceid=303, filesize=12.0 K 2024-11-24T11:15:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,048 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d57999a7f23de1f386379af6f2171765 in 1751ms, sequenceid=303, compaction requested=false 2024-11-24T11:15:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-24T11:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-24T11:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T11:15:01,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8230 sec 2024-11-24T11:15:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.8280 sec 2024-11-24T11:15:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:01,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T11:15:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112490b3d68bfcdb4f1094e21e7232b8785b_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446899513/Put/seqid=0 2024-11-24T11:15:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742009_1185 (size=12454) 2024-11-24T11:15:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,719 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112490b3d68bfcdb4f1094e21e7232b8785b_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112490b3d68bfcdb4f1094e21e7232b8785b_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,720 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/fe873bd7539a4614a9926bc63dad1c95, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/fe873bd7539a4614a9926bc63dad1c95 is 175, key is test_row_0/A:col10/1732446899513/Put/seqid=0 2024-11-24T11:15:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742010_1186 (size=31251) 2024-11-24T11:15:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,727 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/fe873bd7539a4614a9926bc63dad1c95 2024-11-24T11:15:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/f1c47d514a6c4f33a4e2d1eaa8ee4689 is 50, key is test_row_0/B:col10/1732446899513/Put/seqid=0 2024-11-24T11:15:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742011_1187 (size=9857) 2024-11-24T11:15:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:01,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446901778, max timestamp=1732446901807Num uniques 859; 2024-11-24T11:15:01,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446901778, max timestamp=1732446901807Num uniques 859; 2024-11-24T11:15:01,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:01,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446901778, max timestamp=1732446901807Num uniques 859; 2024-11-24T11:15:01,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:01,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:01,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446961809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:01,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446961964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446962074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/f1c47d514a6c4f33a4e2d1eaa8ee4689 2024-11-24T11:15:02,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a0d86f5012d9448c8aff01c9b9fe2846 is 50, key is test_row_0/C:col10/1732446899513/Put/seqid=0 2024-11-24T11:15:02,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742012_1188 (size=9857) 2024-11-24T11:15:02,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446962283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T11:15:02,394 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:02,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-24T11:15:02,397 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T11:15:02,398 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:02,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T11:15:02,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a0d86f5012d9448c8aff01c9b9fe2846 2024-11-24T11:15:02,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/fe873bd7539a4614a9926bc63dad1c95 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fe873bd7539a4614a9926bc63dad1c95 2024-11-24T11:15:02,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fe873bd7539a4614a9926bc63dad1c95, entries=150, sequenceid=317, filesize=30.5 K 2024-11-24T11:15:02,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/f1c47d514a6c4f33a4e2d1eaa8ee4689 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/f1c47d514a6c4f33a4e2d1eaa8ee4689 2024-11-24T11:15:02,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/f1c47d514a6c4f33a4e2d1eaa8ee4689, entries=100, sequenceid=317, filesize=9.6 K 2024-11-24T11:15:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/a0d86f5012d9448c8aff01c9b9fe2846 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a0d86f5012d9448c8aff01c9b9fe2846 2024-11-24T11:15:02,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a0d86f5012d9448c8aff01c9b9fe2846, entries=100, sequenceid=317, filesize=9.6 K 2024-11-24T11:15:02,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446962594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d57999a7f23de1f386379af6f2171765 in 908ms, sequenceid=317, compaction requested=true 2024-11-24T11:15:02,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:02,597 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:02,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:02,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:02,599 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:02,599 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94137 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:02,599 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/A is initiating minor compaction (all files) 2024-11-24T11:15:02,599 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/A in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,600 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/4c733a14bb9b4fdd964e24ae42ff413c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9e60ec57d19045fe9fc794a399e4b2b0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fe873bd7539a4614a9926bc63dad1c95] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=91.9 K 2024-11-24T11:15:02,600 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,600 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/4c733a14bb9b4fdd964e24ae42ff413c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9e60ec57d19045fe9fc794a399e4b2b0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fe873bd7539a4614a9926bc63dad1c95] 2024-11-24T11:15:02,600 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c733a14bb9b4fdd964e24ae42ff413c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732446897317 2024-11-24T11:15:02,602 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:02,602 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/B is initiating minor compaction (all files) 2024-11-24T11:15:02,602 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/B in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,602 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/e63c6b4da78d41c2922a96f3c181a7f0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ed45e2a576a74ab6b2d033bfd96d83e6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/f1c47d514a6c4f33a4e2d1eaa8ee4689] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=34.0 K 2024-11-24T11:15:02,602 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e60ec57d19045fe9fc794a399e4b2b0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732446898459 2024-11-24T11:15:02,603 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e63c6b4da78d41c2922a96f3c181a7f0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732446897317 2024-11-24T11:15:02,603 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe873bd7539a4614a9926bc63dad1c95, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732446899508 2024-11-24T11:15:02,603 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting ed45e2a576a74ab6b2d033bfd96d83e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732446898459 2024-11-24T11:15:02,604 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c47d514a6c4f33a4e2d1eaa8ee4689, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732446899513 2024-11-24T11:15:02,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:02,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:02,614 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:02,622 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#B#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:02,623 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/da2c0f31462e426483d8e1b76db5f18e is 50, key is test_row_0/B:col10/1732446899513/Put/seqid=0 2024-11-24T11:15:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:02,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T11:15:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:02,629 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124f157c57d13c046cbae8ce982a64d4c8f_d57999a7f23de1f386379af6f2171765 store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:02,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:02,631 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124f157c57d13c046cbae8ce982a64d4c8f_d57999a7f23de1f386379af6f2171765, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:02,631 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f157c57d13c046cbae8ce982a64d4c8f_d57999a7f23de1f386379af6f2171765 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:02,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247073a5f0b61a493681eb203a062b1c0b_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446901957/Put/seqid=0 2024-11-24T11:15:02,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742013_1189 (size=12779) 2024-11-24T11:15:02,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742014_1190 (size=4469) 2024-11-24T11:15:02,673 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/da2c0f31462e426483d8e1b76db5f18e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/da2c0f31462e426483d8e1b76db5f18e 2024-11-24T11:15:02,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446962671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446962674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,682 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/B of d57999a7f23de1f386379af6f2171765 into da2c0f31462e426483d8e1b76db5f18e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:02,682 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:02,682 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/B, priority=13, startTime=1732446902599; duration=0sec 2024-11-24T11:15:02,682 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:02,682 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:B 2024-11-24T11:15:02,682 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:02,684 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:02,684 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/C is initiating minor compaction (all files) 2024-11-24T11:15:02,684 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/C in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,684 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d92c0dbb0c0496f9228f73d29f962c0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/e94d4dcad50e4a4e946ea2019f338cce, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a0d86f5012d9448c8aff01c9b9fe2846] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=34.0 K 2024-11-24T11:15:02,685 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d92c0dbb0c0496f9228f73d29f962c0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732446897317 2024-11-24T11:15:02,685 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e94d4dcad50e4a4e946ea2019f338cce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732446898459 2024-11-24T11:15:02,685 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d86f5012d9448c8aff01c9b9fe2846, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732446899513 2024-11-24T11:15:02,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742015_1191 (size=14994) 2024-11-24T11:15:02,702 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:02,703 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/c89b963ed9a6475484f1e2b403fc029a is 50, key is test_row_0/C:col10/1732446899513/Put/seqid=0 2024-11-24T11:15:02,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:02,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:02,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T11:15:02,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742016_1192 (size=12779) 2024-11-24T11:15:02,726 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/c89b963ed9a6475484f1e2b403fc029a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/c89b963ed9a6475484f1e2b403fc029a 2024-11-24T11:15:02,732 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/C of d57999a7f23de1f386379af6f2171765 into c89b963ed9a6475484f1e2b403fc029a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:02,732 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:02,732 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/C, priority=13, startTime=1732446902605; duration=0sec 2024-11-24T11:15:02,732 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:02,732 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:C 2024-11-24T11:15:02,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446962784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446962784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:02,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:02,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446962993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446962994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:03,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:03,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:03,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T11:15:03,074 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#A#compaction#159 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:03,074 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/9638c88548ea4aefa206cdaaab01cb67 is 175, key is test_row_0/A:col10/1732446899513/Put/seqid=0 2024-11-24T11:15:03,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742017_1193 (size=31840) 2024-11-24T11:15:03,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:03,092 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247073a5f0b61a493681eb203a062b1c0b_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247073a5f0b61a493681eb203a062b1c0b_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:03,092 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/46e928bb8aad4730bc76ee5fdad6f450, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:03,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/46e928bb8aad4730bc76ee5fdad6f450 is 175, key is test_row_0/A:col10/1732446901957/Put/seqid=0 2024-11-24T11:15:03,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742018_1194 (size=39949) 2024-11-24T11:15:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446963104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:03,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:03,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446963303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446963304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:03,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:03,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,483 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/9638c88548ea4aefa206cdaaab01cb67 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9638c88548ea4aefa206cdaaab01cb67 2024-11-24T11:15:03,488 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/A of d57999a7f23de1f386379af6f2171765 into 9638c88548ea4aefa206cdaaab01cb67(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:03,488 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:03,488 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/A, priority=13, startTime=1732446902597; duration=0sec 2024-11-24T11:15:03,489 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:03,489 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:A 2024-11-24T11:15:03,499 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/46e928bb8aad4730bc76ee5fdad6f450 2024-11-24T11:15:03,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/dcd1f451746747a2a3e26767147bca22 is 50, key is test_row_0/B:col10/1732446901957/Put/seqid=0 2024-11-24T11:15:03,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742019_1195 (size=12301) 2024-11-24T11:15:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T11:15:03,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:03,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446963814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446963814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:03,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/dcd1f451746747a2a3e26767147bca22 2024-11-24T11:15:03,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/303ec08f14c7429a9d8923fa55b7fa4f is 50, key is test_row_0/C:col10/1732446901957/Put/seqid=0 2024-11-24T11:15:03,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742020_1196 (size=12301) 2024-11-24T11:15:03,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:03,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:04,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:04,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:04,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446964113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:04,116 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:04,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:04,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:04,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:04,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:04,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:04,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/303ec08f14c7429a9d8923fa55b7fa4f 2024-11-24T11:15:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/46e928bb8aad4730bc76ee5fdad6f450 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/46e928bb8aad4730bc76ee5fdad6f450 2024-11-24T11:15:04,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/46e928bb8aad4730bc76ee5fdad6f450, entries=200, sequenceid=352, filesize=39.0 K 2024-11-24T11:15:04,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/dcd1f451746747a2a3e26767147bca22 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/dcd1f451746747a2a3e26767147bca22 2024-11-24T11:15:04,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/dcd1f451746747a2a3e26767147bca22, entries=150, sequenceid=352, filesize=12.0 K 2024-11-24T11:15:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/303ec08f14c7429a9d8923fa55b7fa4f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/303ec08f14c7429a9d8923fa55b7fa4f 2024-11-24T11:15:04,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/303ec08f14c7429a9d8923fa55b7fa4f, entries=150, sequenceid=352, filesize=12.0 K 2024-11-24T11:15:04,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d57999a7f23de1f386379af6f2171765 in 1717ms, sequenceid=352, compaction requested=false 2024-11-24T11:15:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:04,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T11:15:04,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:04,388 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:04,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cd95716b0c384526aa5b6f8124c0d200_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446902668/Put/seqid=0 2024-11-24T11:15:04,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742021_1197 (size=12454) 2024-11-24T11:15:04,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:04,428 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cd95716b0c384526aa5b6f8124c0d200_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd95716b0c384526aa5b6f8124c0d200_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:04,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/88d9ab7efbc64c3f90d89f1358afa482, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:04,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/88d9ab7efbc64c3f90d89f1358afa482 is 175, key is test_row_0/A:col10/1732446902668/Put/seqid=0 2024-11-24T11:15:04,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742022_1198 (size=31255) 2024-11-24T11:15:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T11:15:04,835 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/88d9ab7efbc64c3f90d89f1358afa482 2024-11-24T11:15:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:04,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:04,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/a946d42b7b8545cb91bcda2e52432fb3 is 50, key is test_row_0/B:col10/1732446902668/Put/seqid=0 2024-11-24T11:15:04,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742023_1199 (size=12301) 2024-11-24T11:15:04,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446904858, max timestamp=1732446904881Num uniques 859; 2024-11-24T11:15:04,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446904858, max timestamp=1732446904881Num uniques 859; 2024-11-24T11:15:04,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:04,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:04,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446904858, max timestamp=1732446904881Num uniques 859; 2024-11-24T11:15:04,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:04,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:04,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:04,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446964932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:04,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446964932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446965044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446965044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,249 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/a946d42b7b8545cb91bcda2e52432fb3 2024-11-24T11:15:05,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/21dc84d14ba94ed0b9f6fc583cf204d5 is 50, key is test_row_0/C:col10/1732446902668/Put/seqid=0 2024-11-24T11:15:05,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446965253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446965254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742024_1200 (size=12301) 2024-11-24T11:15:05,263 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/21dc84d14ba94ed0b9f6fc583cf204d5 2024-11-24T11:15:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/88d9ab7efbc64c3f90d89f1358afa482 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/88d9ab7efbc64c3f90d89f1358afa482 2024-11-24T11:15:05,271 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/88d9ab7efbc64c3f90d89f1358afa482, entries=150, sequenceid=365, filesize=30.5 K 2024-11-24T11:15:05,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/a946d42b7b8545cb91bcda2e52432fb3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a946d42b7b8545cb91bcda2e52432fb3 2024-11-24T11:15:05,277 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a946d42b7b8545cb91bcda2e52432fb3, entries=150, sequenceid=365, filesize=12.0 K 2024-11-24T11:15:05,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/21dc84d14ba94ed0b9f6fc583cf204d5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/21dc84d14ba94ed0b9f6fc583cf204d5 2024-11-24T11:15:05,282 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/21dc84d14ba94ed0b9f6fc583cf204d5, entries=150, sequenceid=365, filesize=12.0 K 2024-11-24T11:15:05,283 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d57999a7f23de1f386379af6f2171765 in 895ms, sequenceid=365, compaction requested=true 2024-11-24T11:15:05,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:05,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:05,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-24T11:15:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-24T11:15:05,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-24T11:15:05,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8870 sec 2024-11-24T11:15:05,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8930 sec 2024-11-24T11:15:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:05,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T11:15:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:05,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446965599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446965599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243a08b34c53354e0b8823edf4625e3fde_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446904930/Put/seqid=0 2024-11-24T11:15:05,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742025_1201 (size=14994) 2024-11-24T11:15:05,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446965713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446965713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446965923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446965923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:06,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243a08b34c53354e0b8823edf4625e3fde_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243a08b34c53354e0b8823edf4625e3fde_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:06,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/43e52eaf79164a98b8c1ac91efa0b478, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:06,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/43e52eaf79164a98b8c1ac91efa0b478 is 175, key is test_row_0/A:col10/1732446904930/Put/seqid=0 2024-11-24T11:15:06,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742026_1202 (size=39949) 2024-11-24T11:15:06,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446966134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:06,135 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446966234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446966234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:06,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/43e52eaf79164a98b8c1ac91efa0b478 2024-11-24T11:15:06,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/d3150ce9d0cf4b4aa238b0e4c141ba25 is 50, key is test_row_0/B:col10/1732446904930/Put/seqid=0 2024-11-24T11:15:06,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742027_1203 (size=12301) 2024-11-24T11:15:06,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/d3150ce9d0cf4b4aa238b0e4c141ba25 2024-11-24T11:15:06,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/2cc55b3939014466aab72f970f7cacce is 50, key is test_row_0/C:col10/1732446904930/Put/seqid=0 2024-11-24T11:15:06,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742028_1204 (size=12301) 2024-11-24T11:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T11:15:06,574 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:06,574 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-24T11:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T11:15:06,577 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:06,578 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:06,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T11:15:06,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T11:15:06,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:06,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446966744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446966744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:06,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T11:15:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:06,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:06,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/2cc55b3939014466aab72f970f7cacce 2024-11-24T11:15:06,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/43e52eaf79164a98b8c1ac91efa0b478 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/43e52eaf79164a98b8c1ac91efa0b478 2024-11-24T11:15:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T11:15:06,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/43e52eaf79164a98b8c1ac91efa0b478, entries=200, sequenceid=402, filesize=39.0 K 2024-11-24T11:15:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/d3150ce9d0cf4b4aa238b0e4c141ba25 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/d3150ce9d0cf4b4aa238b0e4c141ba25 2024-11-24T11:15:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/d3150ce9d0cf4b4aa238b0e4c141ba25, entries=150, sequenceid=402, filesize=12.0 K 2024-11-24T11:15:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/2cc55b3939014466aab72f970f7cacce as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2cc55b3939014466aab72f970f7cacce 2024-11-24T11:15:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2cc55b3939014466aab72f970f7cacce, entries=150, sequenceid=402, filesize=12.0 K 2024-11-24T11:15:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for d57999a7f23de1f386379af6f2171765 in 1326ms, sequenceid=402, compaction requested=true 2024-11-24T11:15:06,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:06,908 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T11:15:06,908 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,909 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142993 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:06,909 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:06,909 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/B is initiating minor compaction (all files) 2024-11-24T11:15:06,909 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/A is initiating minor compaction (all files) 2024-11-24T11:15:06,909 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/A in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:06,909 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/B in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:06,910 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9638c88548ea4aefa206cdaaab01cb67, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/46e928bb8aad4730bc76ee5fdad6f450, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/88d9ab7efbc64c3f90d89f1358afa482, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/43e52eaf79164a98b8c1ac91efa0b478] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=139.6 K 2024-11-24T11:15:06,910 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/da2c0f31462e426483d8e1b76db5f18e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/dcd1f451746747a2a3e26767147bca22, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a946d42b7b8545cb91bcda2e52432fb3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/d3150ce9d0cf4b4aa238b0e4c141ba25] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=48.5 K 2024-11-24T11:15:06,910 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:06,910 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9638c88548ea4aefa206cdaaab01cb67, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/46e928bb8aad4730bc76ee5fdad6f450, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/88d9ab7efbc64c3f90d89f1358afa482, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/43e52eaf79164a98b8c1ac91efa0b478] 2024-11-24T11:15:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,910 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting da2c0f31462e426483d8e1b76db5f18e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732446898467 2024-11-24T11:15:06,910 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9638c88548ea4aefa206cdaaab01cb67, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732446898467 2024-11-24T11:15:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,911 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e928bb8aad4730bc76ee5fdad6f450, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732446901947 2024-11-24T11:15:06,911 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting dcd1f451746747a2a3e26767147bca22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732446901957 2024-11-24T11:15:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,912 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a946d42b7b8545cb91bcda2e52432fb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732446902635 2024-11-24T11:15:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,912 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d9ab7efbc64c3f90d89f1358afa482, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732446902635 2024-11-24T11:15:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,913 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d3150ce9d0cf4b4aa238b0e4c141ba25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732446904930 2024-11-24T11:15:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,913 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e52eaf79164a98b8c1ac91efa0b478, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732446904904 2024-11-24T11:15:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,933 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,943 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:06,943 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/8c27f023bdb045f18988008bef5fb367 is 50, key is test_row_0/B:col10/1732446904930/Put/seqid=0 2024-11-24T11:15:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,949 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411248f164cf969b4489bb079f09ab2f01d66_d57999a7f23de1f386379af6f2171765 store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:06,950 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411248f164cf969b4489bb079f09ab2f01d66_d57999a7f23de1f386379af6f2171765, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:06,950 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248f164cf969b4489bb079f09ab2f01d66_d57999a7f23de1f386379af6f2171765 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742029_1205 (size=12915) 2024-11-24T11:15:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742030_1206 (size=4469) 2024-11-24T11:15:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,994 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#A#compaction#171 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,994 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/df20b3efe3364069b9ba40d85baaba01 is 175, key is test_row_0/A:col10/1732446904930/Put/seqid=0 2024-11-24T11:15:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742031_1207 (size=31869) 2024-11-24T11:15:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T11:15:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:07,037 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bc119096b0054a5f916a1d7a2c581ed8_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446905598/Put/seqid=0 2024-11-24T11:15:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742032_1208 (size=9914) 2024-11-24T11:15:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,084 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bc119096b0054a5f916a1d7a2c581ed8_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bc119096b0054a5f916a1d7a2c581ed8_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/93283475c3e84c03a2088147eb7a6a3a, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:07,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/93283475c3e84c03a2088147eb7a6a3a is 175, key is test_row_0/A:col10/1732446905598/Put/seqid=0 2024-11-24T11:15:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742033_1209 (size=22561) 2024-11-24T11:15:07,097 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/93283475c3e84c03a2088147eb7a6a3a 2024-11-24T11:15:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/b95b533d74ba4d9b98bff6af169a714d is 50, key is test_row_0/B:col10/1732446905598/Put/seqid=0 2024-11-24T11:15:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742034_1210 (size=9857) 2024-11-24T11:15:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T11:15:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,391 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/8c27f023bdb045f18988008bef5fb367 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8c27f023bdb045f18988008bef5fb367 2024-11-24T11:15:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,397 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/B of d57999a7f23de1f386379af6f2171765 into 8c27f023bdb045f18988008bef5fb367(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:07,398 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:07,398 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/B, priority=12, startTime=1732446906908; duration=0sec 2024-11-24T11:15:07,398 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:07,398 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:B 2024-11-24T11:15:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,398 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,401 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,403 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/C is initiating minor compaction (all files) 2024-11-24T11:15:07,404 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/C in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,404 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/c89b963ed9a6475484f1e2b403fc029a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/303ec08f14c7429a9d8923fa55b7fa4f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/21dc84d14ba94ed0b9f6fc583cf204d5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2cc55b3939014466aab72f970f7cacce] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=48.5 K 2024-11-24T11:15:07,404 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c89b963ed9a6475484f1e2b403fc029a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732446898467 2024-11-24T11:15:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,407 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 303ec08f14c7429a9d8923fa55b7fa4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732446901957 2024-11-24T11:15:07,407 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 21dc84d14ba94ed0b9f6fc583cf204d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732446902635 2024-11-24T11:15:07,407 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/df20b3efe3364069b9ba40d85baaba01 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/df20b3efe3364069b9ba40d85baaba01 2024-11-24T11:15:07,408 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc55b3939014466aab72f970f7cacce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732446904930 2024-11-24T11:15:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,413 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/A of d57999a7f23de1f386379af6f2171765 into df20b3efe3364069b9ba40d85baaba01(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:07,413 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:07,413 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/A, priority=12, startTime=1732446906908; duration=0sec 2024-11-24T11:15:07,413 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:07,413 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:A 2024-11-24T11:15:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,422 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#C#compaction#175 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:07,422 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/42bed193696f4d9898234c5392e95985 is 50, key is test_row_0/C:col10/1732446904930/Put/seqid=0 2024-11-24T11:15:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742035_1211 (size=12915) 2024-11-24T11:15:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,449 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/42bed193696f4d9898234c5392e95985 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/42bed193696f4d9898234c5392e95985 2024-11-24T11:15:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,456 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d57999a7f23de1f386379af6f2171765/C of d57999a7f23de1f386379af6f2171765 into 42bed193696f4d9898234c5392e95985(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:07,456 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:07,456 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/C, priority=12, startTime=1732446906908; duration=0sec 2024-11-24T11:15:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,457 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,457 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:C 2024-11-24T11:15:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,550 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/b95b533d74ba4d9b98bff6af169a714d 2024-11-24T11:15:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/72b4c8408ebc40caa96036a1f4b8613d is 50, key is test_row_0/C:col10/1732446905598/Put/seqid=0 2024-11-24T11:15:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742036_1212 (size=9857) 2024-11-24T11:15:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T11:15:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:07,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:07,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446907796, max timestamp=1732446907811Num uniques 859; 2024-11-24T11:15:07,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:07,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446907796, max timestamp=1732446907811Num uniques 859; 2024-11-24T11:15:07,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446907796, max timestamp=1732446907811Num uniques 859; 2024-11-24T11:15:07,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:07,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:07,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446967842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:07,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446967843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:07,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446967953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446967963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:07,968 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/72b4c8408ebc40caa96036a1f4b8613d 2024-11-24T11:15:07,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/93283475c3e84c03a2088147eb7a6a3a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/93283475c3e84c03a2088147eb7a6a3a 2024-11-24T11:15:07,977 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/93283475c3e84c03a2088147eb7a6a3a, entries=100, sequenceid=412, filesize=22.0 K 2024-11-24T11:15:07,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/b95b533d74ba4d9b98bff6af169a714d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b95b533d74ba4d9b98bff6af169a714d 2024-11-24T11:15:07,982 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b95b533d74ba4d9b98bff6af169a714d, entries=100, sequenceid=412, filesize=9.6 K 2024-11-24T11:15:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/72b4c8408ebc40caa96036a1f4b8613d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/72b4c8408ebc40caa96036a1f4b8613d 2024-11-24T11:15:07,987 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/72b4c8408ebc40caa96036a1f4b8613d, entries=100, sequenceid=412, filesize=9.6 K 2024-11-24T11:15:07,988 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d57999a7f23de1f386379af6f2171765 in 951ms, sequenceid=412, compaction requested=false 2024-11-24T11:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-24T11:15:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-24T11:15:07,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-24T11:15:07,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-24T11:15:07,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4170 sec 2024-11-24T11:15:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:08,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-24T11:15:08,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:08,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:08,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:08,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:08,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:08,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:08,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249930a06b366e4ab59b9321eaf39586bd_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446908167/Put/seqid=0 2024-11-24T11:15:08,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742037_1213 (size=14994) 2024-11-24T11:15:08,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:08,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446968191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:08,196 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249930a06b366e4ab59b9321eaf39586bd_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249930a06b366e4ab59b9321eaf39586bd_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:08,197 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/84fb5aa17eca4993babc308e3de00237, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:08,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/84fb5aa17eca4993babc308e3de00237 is 175, key is test_row_0/A:col10/1732446908167/Put/seqid=0 2024-11-24T11:15:08,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446968193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:08,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742038_1214 (size=39949) 2024-11-24T11:15:08,221 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=451, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/84fb5aa17eca4993babc308e3de00237 2024-11-24T11:15:08,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/ee1482283dac4d52a34d3d296f40b860 is 50, key is test_row_0/B:col10/1732446908167/Put/seqid=0 2024-11-24T11:15:08,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742039_1215 (size=12301) 2024-11-24T11:15:08,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/ee1482283dac4d52a34d3d296f40b860 2024-11-24T11:15:08,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/59858aa16faa4fdabc44a8ab4e1ae31d is 50, key is test_row_0/C:col10/1732446908167/Put/seqid=0 2024-11-24T11:15:08,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742040_1216 (size=12301) 2024-11-24T11:15:08,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446968303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446968303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:08,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446968513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446968513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:08,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/59858aa16faa4fdabc44a8ab4e1ae31d 2024-11-24T11:15:08,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/84fb5aa17eca4993babc308e3de00237 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/84fb5aa17eca4993babc308e3de00237 2024-11-24T11:15:08,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/84fb5aa17eca4993babc308e3de00237, entries=200, sequenceid=451, filesize=39.0 K 2024-11-24T11:15:08,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/ee1482283dac4d52a34d3d296f40b860 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ee1482283dac4d52a34d3d296f40b860 2024-11-24T11:15:08,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ee1482283dac4d52a34d3d296f40b860, entries=150, sequenceid=451, filesize=12.0 K 2024-11-24T11:15:08,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/59858aa16faa4fdabc44a8ab4e1ae31d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/59858aa16faa4fdabc44a8ab4e1ae31d 2024-11-24T11:15:08,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/59858aa16faa4fdabc44a8ab4e1ae31d, entries=150, sequenceid=451, filesize=12.0 K 2024-11-24T11:15:08,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for d57999a7f23de1f386379af6f2171765 in 511ms, sequenceid=451, compaction requested=true 2024-11-24T11:15:08,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:08,680 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:08,680 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:08,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:08,681 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:08,681 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:08,681 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/A is initiating minor compaction (all files) 2024-11-24T11:15:08,681 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/B is initiating minor compaction (all files) 2024-11-24T11:15:08,681 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/B in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:08,681 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/A in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:08,681 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8c27f023bdb045f18988008bef5fb367, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b95b533d74ba4d9b98bff6af169a714d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ee1482283dac4d52a34d3d296f40b860] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=34.3 K 2024-11-24T11:15:08,681 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/df20b3efe3364069b9ba40d85baaba01, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/93283475c3e84c03a2088147eb7a6a3a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/84fb5aa17eca4993babc308e3de00237] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=92.2 K 2024-11-24T11:15:08,681 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:08,681 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/df20b3efe3364069b9ba40d85baaba01, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/93283475c3e84c03a2088147eb7a6a3a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/84fb5aa17eca4993babc308e3de00237] 2024-11-24T11:15:08,681 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c27f023bdb045f18988008bef5fb367, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732446904930 2024-11-24T11:15:08,682 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting df20b3efe3364069b9ba40d85baaba01, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732446904930 2024-11-24T11:15:08,682 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b95b533d74ba4d9b98bff6af169a714d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732446905587 2024-11-24T11:15:08,682 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93283475c3e84c03a2088147eb7a6a3a, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732446905587 2024-11-24T11:15:08,682 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1482283dac4d52a34d3d296f40b860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732446907835 2024-11-24T11:15:08,682 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84fb5aa17eca4993babc308e3de00237, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732446907835 2024-11-24T11:15:08,690 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:08,691 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/b7f2988d66e64755840a88bc9fd879b7 is 50, key is test_row_0/B:col10/1732446908167/Put/seqid=0 2024-11-24T11:15:08,692 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:08,696 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124ca3b0b51ec3b4aba9548e787001b3538_d57999a7f23de1f386379af6f2171765 store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:08,698 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124ca3b0b51ec3b4aba9548e787001b3538_d57999a7f23de1f386379af6f2171765, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:08,698 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ca3b0b51ec3b4aba9548e787001b3538_d57999a7f23de1f386379af6f2171765 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:08,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742041_1217 (size=13017) 2024-11-24T11:15:08,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742042_1218 (size=4469) 2024-11-24T11:15:08,706 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/b7f2988d66e64755840a88bc9fd879b7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b7f2988d66e64755840a88bc9fd879b7 2024-11-24T11:15:08,712 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/B of d57999a7f23de1f386379af6f2171765 into b7f2988d66e64755840a88bc9fd879b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:08,712 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:08,712 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/B, priority=13, startTime=1732446908680; duration=0sec 2024-11-24T11:15:08,712 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:08,712 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:B 2024-11-24T11:15:08,712 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:08,713 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:08,713 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/C is initiating minor compaction (all files) 2024-11-24T11:15:08,714 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/C in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:08,714 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/42bed193696f4d9898234c5392e95985, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/72b4c8408ebc40caa96036a1f4b8613d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/59858aa16faa4fdabc44a8ab4e1ae31d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=34.3 K 2024-11-24T11:15:08,714 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 42bed193696f4d9898234c5392e95985, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732446904930 2024-11-24T11:15:08,714 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b4c8408ebc40caa96036a1f4b8613d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732446905587 2024-11-24T11:15:08,715 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 59858aa16faa4fdabc44a8ab4e1ae31d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732446907835 2024-11-24T11:15:08,722 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:08,722 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/2e95d28b19fc4a98a271fbf2a3616527 is 50, key is test_row_0/C:col10/1732446908167/Put/seqid=0 2024-11-24T11:15:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T11:15:08,723 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:08,724 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-24T11:15:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:08,726 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:08,726 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:08,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:08,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742043_1219 (size=13017) 2024-11-24T11:15:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:08,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:08,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:08,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:08,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:08,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:08,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:08,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:08,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124921206f98418414a8299b3e9b73f37db_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446908191/Put/seqid=0 2024-11-24T11:15:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:08,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742044_1220 (size=14994) 2024-11-24T11:15:08,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:08,841 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124921206f98418414a8299b3e9b73f37db_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124921206f98418414a8299b3e9b73f37db_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:08,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/6e9122b162764977a6068f1c5388658e, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:08,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/6e9122b162764977a6068f1c5388658e is 175, key is test_row_0/A:col10/1732446908191/Put/seqid=0 2024-11-24T11:15:08,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742045_1221 (size=39949) 2024-11-24T11:15:08,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446908858, max timestamp=1732446908874Num uniques 859; 2024-11-24T11:15:08,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446908858, max timestamp=1732446908874Num uniques 859; 2024-11-24T11:15:08,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:08,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:08,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446908858, max timestamp=1732446908874Num uniques 859; 2024-11-24T11:15:08,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:08,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:08,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446968911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:08,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446968912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446969024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446969024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:09,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:09,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:09,105 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:09,105 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/00315eafb19c4898869d14b7a83034b6 is 175, key is test_row_0/A:col10/1732446908167/Put/seqid=0 2024-11-24T11:15:09,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742046_1222 (size=31971) 2024-11-24T11:15:09,113 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/00315eafb19c4898869d14b7a83034b6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/00315eafb19c4898869d14b7a83034b6 2024-11-24T11:15:09,117 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/A of d57999a7f23de1f386379af6f2171765 into 00315eafb19c4898869d14b7a83034b6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:09,118 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:09,118 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/A, priority=13, startTime=1732446908680; duration=0sec 2024-11-24T11:15:09,118 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:09,118 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:A 2024-11-24T11:15:09,135 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/2e95d28b19fc4a98a271fbf2a3616527 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2e95d28b19fc4a98a271fbf2a3616527 2024-11-24T11:15:09,140 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/C of d57999a7f23de1f386379af6f2171765 into 2e95d28b19fc4a98a271fbf2a3616527(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:09,140 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:09,140 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/C, priority=13, startTime=1732446908680; duration=0sec 2024-11-24T11:15:09,140 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:09,140 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:C 2024-11-24T11:15:09,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:09,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:09,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42104 deadline: 1732446969184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,188 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446969233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446969233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42148 deadline: 1732446969243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,244 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18354 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,249 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=462, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/6e9122b162764977a6068f1c5388658e 2024-11-24T11:15:09,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/03253ec4a6b94fb2843c24855ffac2d3 is 50, key is test_row_0/B:col10/1732446908191/Put/seqid=0 2024-11-24T11:15:09,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742047_1223 (size=12301) 2024-11-24T11:15:09,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:09,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:09,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446969543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446969543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:09,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/03253ec4a6b94fb2843c24855ffac2d3 2024-11-24T11:15:09,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/04842d44ce724012bc99440acb8f765f is 50, key is test_row_0/C:col10/1732446908191/Put/seqid=0 2024-11-24T11:15:09,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742048_1224 (size=12301) 2024-11-24T11:15:09,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:09,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:09,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:09,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:09,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446970054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:10,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446970063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:10,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/04842d44ce724012bc99440acb8f765f 2024-11-24T11:15:10,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/6e9122b162764977a6068f1c5388658e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/6e9122b162764977a6068f1c5388658e 2024-11-24T11:15:10,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/6e9122b162764977a6068f1c5388658e, entries=200, sequenceid=462, filesize=39.0 K 2024-11-24T11:15:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/03253ec4a6b94fb2843c24855ffac2d3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/03253ec4a6b94fb2843c24855ffac2d3 2024-11-24T11:15:10,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/03253ec4a6b94fb2843c24855ffac2d3, entries=150, sequenceid=462, filesize=12.0 K 2024-11-24T11:15:10,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/04842d44ce724012bc99440acb8f765f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/04842d44ce724012bc99440acb8f765f 2024-11-24T11:15:10,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:10,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/04842d44ce724012bc99440acb8f765f, entries=150, sequenceid=462, filesize=12.0 K 2024-11-24T11:15:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d57999a7f23de1f386379af6f2171765 in 1277ms, sequenceid=462, compaction requested=false 2024-11-24T11:15:10,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:10,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T11:15:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:10,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247ed03fc2d9674da0bf361dc1655704aa_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446908900/Put/seqid=0 2024-11-24T11:15:10,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742049_1225 (size=14994) 2024-11-24T11:15:10,202 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:10,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247ed03fc2d9674da0bf361dc1655704aa_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247ed03fc2d9674da0bf361dc1655704aa_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:10,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/1ba202e060cf40f0a1672b1691849a0e, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:10,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/1ba202e060cf40f0a1672b1691849a0e is 175, key is test_row_0/A:col10/1732446908900/Put/seqid=0 2024-11-24T11:15:10,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742050_1226 (size=39949) 2024-11-24T11:15:10,222 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=501, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/1ba202e060cf40f0a1672b1691849a0e 2024-11-24T11:15:10,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446970215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:10,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/1512473395d4460cb60844435fb3d537 is 50, key is test_row_0/B:col10/1732446908900/Put/seqid=0 2024-11-24T11:15:10,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742051_1227 (size=12301) 2024-11-24T11:15:10,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/1512473395d4460cb60844435fb3d537 2024-11-24T11:15:10,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/7d707567d48c42a2ba06872e820114ef is 50, key is test_row_0/C:col10/1732446908900/Put/seqid=0 2024-11-24T11:15:10,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742052_1228 (size=12301) 2024-11-24T11:15:10,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/7d707567d48c42a2ba06872e820114ef 2024-11-24T11:15:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/1ba202e060cf40f0a1672b1691849a0e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/1ba202e060cf40f0a1672b1691849a0e 2024-11-24T11:15:10,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/1ba202e060cf40f0a1672b1691849a0e, entries=200, sequenceid=501, filesize=39.0 K 2024-11-24T11:15:10,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/1512473395d4460cb60844435fb3d537 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/1512473395d4460cb60844435fb3d537 2024-11-24T11:15:10,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/1512473395d4460cb60844435fb3d537, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T11:15:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/7d707567d48c42a2ba06872e820114ef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d707567d48c42a2ba06872e820114ef 2024-11-24T11:15:10,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d707567d48c42a2ba06872e820114ef, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T11:15:10,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for d57999a7f23de1f386379af6f2171765 in 122ms, sequenceid=501, compaction requested=true 2024-11-24T11:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:10,299 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:10,299 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57999a7f23de1f386379af6f2171765:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:10,301 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:10,302 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/A is initiating minor compaction (all files) 2024-11-24T11:15:10,302 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/A in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,302 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/00315eafb19c4898869d14b7a83034b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/6e9122b162764977a6068f1c5388658e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/1ba202e060cf40f0a1672b1691849a0e] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=109.2 K 2024-11-24T11:15:10,302 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,302 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/00315eafb19c4898869d14b7a83034b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/6e9122b162764977a6068f1c5388658e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/1ba202e060cf40f0a1672b1691849a0e] 2024-11-24T11:15:10,302 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00315eafb19c4898869d14b7a83034b6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732446907835 2024-11-24T11:15:10,303 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9122b162764977a6068f1c5388658e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1732446908170 2024-11-24T11:15:10,303 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ba202e060cf40f0a1672b1691849a0e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732446908891 2024-11-24T11:15:10,304 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:10,304 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/B is initiating minor compaction (all files) 2024-11-24T11:15:10,304 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/B in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,304 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b7f2988d66e64755840a88bc9fd879b7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/03253ec4a6b94fb2843c24855ffac2d3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/1512473395d4460cb60844435fb3d537] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=36.7 K 2024-11-24T11:15:10,306 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b7f2988d66e64755840a88bc9fd879b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732446907835 2024-11-24T11:15:10,307 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 03253ec4a6b94fb2843c24855ffac2d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1732446908170 2024-11-24T11:15:10,307 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1512473395d4460cb60844435fb3d537, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732446908900 2024-11-24T11:15:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:10,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:10,345 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:10,347 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#B#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:10,348 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/fedb9d6a899b41ba8eb6770e3ee7978f is 50, key is test_row_0/B:col10/1732446908900/Put/seqid=0 2024-11-24T11:15:10,353 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112490a7e8031fc94509b0c1b6b9048611ea_d57999a7f23de1f386379af6f2171765 store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:10,356 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112490a7e8031fc94509b0c1b6b9048611ea_d57999a7f23de1f386379af6f2171765, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:10,356 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112490a7e8031fc94509b0c1b6b9048611ea_d57999a7f23de1f386379af6f2171765 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:10,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fd3e074e43e44985bd2175c14e50c43d_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446910208/Put/seqid=0 2024-11-24T11:15:10,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:10,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:10,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742053_1229 (size=13119) 2024-11-24T11:15:10,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742054_1230 (size=4469) 2024-11-24T11:15:10,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742055_1231 (size=14994) 2024-11-24T11:15:10,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446910415, max timestamp=1732446910467Num uniques 859; 2024-11-24T11:15:10,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:10,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446910415, max timestamp=1732446910467Num uniques 859; 2024-11-24T11:15:10,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:10,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446910415, max timestamp=1732446910467Num uniques 859; 2024-11-24T11:15:10,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:10,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446970576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:10,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446970684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:10,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,760 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-24T11:15:10,760 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:10,760 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,760 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:10,767 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-24T11:15:10,767 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:10,767 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,767 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:10,770 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-24T11:15:10,770 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:10,770 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,771 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:10,778 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-24T11:15:10,778 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:10,778 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,778 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:10,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:10,822 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/fedb9d6a899b41ba8eb6770e3ee7978f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/fedb9d6a899b41ba8eb6770e3ee7978f 2024-11-24T11:15:10,830 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/B of d57999a7f23de1f386379af6f2171765 into fedb9d6a899b41ba8eb6770e3ee7978f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:10,830 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:10,830 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/B, priority=13, startTime=1732446910299; duration=0sec 2024-11-24T11:15:10,830 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:10,830 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:B 2024-11-24T11:15:10,830 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:10,831 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:10,831 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): d57999a7f23de1f386379af6f2171765/C is initiating minor compaction (all files) 2024-11-24T11:15:10,831 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d57999a7f23de1f386379af6f2171765/C in TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,831 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2e95d28b19fc4a98a271fbf2a3616527, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/04842d44ce724012bc99440acb8f765f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d707567d48c42a2ba06872e820114ef] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp, totalSize=36.7 K 2024-11-24T11:15:10,832 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e95d28b19fc4a98a271fbf2a3616527, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732446907835 2024-11-24T11:15:10,832 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 04842d44ce724012bc99440acb8f765f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1732446908170 2024-11-24T11:15:10,833 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d707567d48c42a2ba06872e820114ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732446908900 2024-11-24T11:15:10,840 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#C#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:10,840 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/b076301b44eb4633bc356e63f1bf2f67 is 50, key is test_row_0/C:col10/1732446908900/Put/seqid=0 2024-11-24T11:15:10,848 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57999a7f23de1f386379af6f2171765#A#compaction#189 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:10,849 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/fef1022958a544b6a1b810c67853a4f5 is 175, key is test_row_0/A:col10/1732446908900/Put/seqid=0 2024-11-24T11:15:10,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742056_1232 (size=13119) 2024-11-24T11:15:10,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742057_1233 (size=32073) 2024-11-24T11:15:10,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:10,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fd3e074e43e44985bd2175c14e50c43d_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fd3e074e43e44985bd2175c14e50c43d_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:10,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:10,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:10,872 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/b076301b44eb4633bc356e63f1bf2f67 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/b076301b44eb4633bc356e63f1bf2f67 2024-11-24T11:15:10,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/911b8cceb0bc4a91a14591d6cef58536, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:10,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/911b8cceb0bc4a91a14591d6cef58536 is 175, key is test_row_0/A:col10/1732446910208/Put/seqid=0 2024-11-24T11:15:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:10,882 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/C of d57999a7f23de1f386379af6f2171765 into b076301b44eb4633bc356e63f1bf2f67(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:10,882 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:10,883 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/C, priority=13, startTime=1732446910299; duration=0sec 2024-11-24T11:15:10,883 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:10,883 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:C 2024-11-24T11:15:10,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742058_1234 (size=39949) 2024-11-24T11:15:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446970893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:10,894 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=511, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/911b8cceb0bc4a91a14591d6cef58536 2024-11-24T11:15:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:10,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/7e75d7e5015f491482eb19aca1cff615 is 50, key is test_row_0/B:col10/1732446910208/Put/seqid=0 2024-11-24T11:15:10,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742059_1235 (size=12301) 2024-11-24T11:15:11,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1732446971074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1732446971073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:11,074 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:11,074 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in d57999a7f23de1f386379af6f2171765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:11,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:11,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:11,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446971204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:11,267 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/fef1022958a544b6a1b810c67853a4f5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fef1022958a544b6a1b810c67853a4f5 2024-11-24T11:15:11,272 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d57999a7f23de1f386379af6f2171765/A of d57999a7f23de1f386379af6f2171765 into fef1022958a544b6a1b810c67853a4f5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:11,273 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:11,273 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., storeName=d57999a7f23de1f386379af6f2171765/A, priority=13, startTime=1732446910299; duration=0sec 2024-11-24T11:15:11,273 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:11,273 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57999a7f23de1f386379af6f2171765:A 2024-11-24T11:15:11,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:11,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/7e75d7e5015f491482eb19aca1cff615 2024-11-24T11:15:11,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/b82ae1d6f08b4ee480478c4eae669469 is 50, key is test_row_0/C:col10/1732446910208/Put/seqid=0 2024-11-24T11:15:11,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742060_1236 (size=12301) 2024-11-24T11:15:11,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:11,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42120 deadline: 1732446971713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:11,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/b82ae1d6f08b4ee480478c4eae669469 2024-11-24T11:15:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/911b8cceb0bc4a91a14591d6cef58536 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/911b8cceb0bc4a91a14591d6cef58536 2024-11-24T11:15:11,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/911b8cceb0bc4a91a14591d6cef58536, entries=200, sequenceid=511, filesize=39.0 K 2024-11-24T11:15:11,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/7e75d7e5015f491482eb19aca1cff615 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/7e75d7e5015f491482eb19aca1cff615 2024-11-24T11:15:11,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/7e75d7e5015f491482eb19aca1cff615, entries=150, sequenceid=511, filesize=12.0 K 2024-11-24T11:15:11,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/b82ae1d6f08b4ee480478c4eae669469 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/b82ae1d6f08b4ee480478c4eae669469 2024-11-24T11:15:11,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/b82ae1d6f08b4ee480478c4eae669469, entries=150, sequenceid=511, filesize=12.0 K 2024-11-24T11:15:11,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d57999a7f23de1f386379af6f2171765 in 1443ms, sequenceid=511, compaction requested=false 2024-11-24T11:15:11,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:11,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T11:15:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:11,793 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T11:15:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:11,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244ac49b4216fa42e086ea28a8f2a6d56e_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446910575/Put/seqid=0 2024-11-24T11:15:11,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742061_1237 (size=12454) 2024-11-24T11:15:12,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:12,213 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244ac49b4216fa42e086ea28a8f2a6d56e_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ac49b4216fa42e086ea28a8f2a6d56e_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:12,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/dde86a33557b4e3f8902d1f41501ba6e, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:12,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/dde86a33557b4e3f8902d1f41501ba6e is 175, key is test_row_0/A:col10/1732446910575/Put/seqid=0 2024-11-24T11:15:12,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742062_1238 (size=31255) 2024-11-24T11:15:12,623 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=550, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/dde86a33557b4e3f8902d1f41501ba6e 2024-11-24T11:15:12,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/981db041e0634b368a07d4fd226eae15 is 50, key is test_row_0/B:col10/1732446910575/Put/seqid=0 2024-11-24T11:15:12,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742063_1239 (size=12301) 2024-11-24T11:15:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:12,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. as already flushing 2024-11-24T11:15:12,725 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-24T11:15:12,725 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:12,725 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:12,725 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:12,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:13,042 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/981db041e0634b368a07d4fd226eae15 2024-11-24T11:15:13,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/77517c4c00214aaba83d993314ac1321 is 50, key is test_row_0/C:col10/1732446910575/Put/seqid=0 2024-11-24T11:15:13,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742064_1240 (size=12301) 2024-11-24T11:15:13,085 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-24T11:15:13,085 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:13,085 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:13,085 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:13,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:13,105 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-24T11:15:13,105 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:13,105 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:13,105 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:13,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:13,452 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/77517c4c00214aaba83d993314ac1321 2024-11-24T11:15:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/dde86a33557b4e3f8902d1f41501ba6e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/dde86a33557b4e3f8902d1f41501ba6e 2024-11-24T11:15:13,465 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/dde86a33557b4e3f8902d1f41501ba6e, entries=150, sequenceid=550, filesize=30.5 K 2024-11-24T11:15:13,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/981db041e0634b368a07d4fd226eae15 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/981db041e0634b368a07d4fd226eae15 2024-11-24T11:15:13,472 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/981db041e0634b368a07d4fd226eae15, entries=150, sequenceid=550, filesize=12.0 K 2024-11-24T11:15:13,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/77517c4c00214aaba83d993314ac1321 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/77517c4c00214aaba83d993314ac1321 2024-11-24T11:15:13,478 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/77517c4c00214aaba83d993314ac1321, entries=150, sequenceid=550, filesize=12.0 K 2024-11-24T11:15:13,478 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=20.13 KB/20610 for d57999a7f23de1f386379af6f2171765 in 1685ms, sequenceid=550, compaction requested=true 2024-11-24T11:15:13,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for d57999a7f23de1f386379af6f2171765: 2024-11-24T11:15:13,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T11:15:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T11:15:13,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T11:15:13,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.7540 sec 2024-11-24T11:15:13,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 4.7570 sec 2024-11-24T11:15:14,519 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T11:15:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T11:15:16,925 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:19,229 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-24T11:15:19,229 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:19,229 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:19,229 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:19,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:19,320 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-24T11:15:19,320 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:19,320 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:19,320 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:19,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:19,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T11:15:19,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-24T11:15:19,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-24T11:15:19,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-24T11:15:19,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 153 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2555 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2624 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1074 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3219 rows 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1075 2024-11-24T11:15:19,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3223 rows 2024-11-24T11:15:19,322 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:15:19,322 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:15:19,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:19,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:19,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:19,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-24T11:15:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T11:15:19,327 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446919327"}]},"ts":"1732446919327"} 2024-11-24T11:15:19,329 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T11:15:19,329 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T11:15:19,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:15:19,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, UNASSIGN}] 2024-11-24T11:15:19,334 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, UNASSIGN 2024-11-24T11:15:19,335 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:19,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, UNASSIGN because future has completed 2024-11-24T11:15:19,337 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:15:19,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:15:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T11:15:19,490 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:19,490 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:15:19,490 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing d57999a7f23de1f386379af6f2171765, disabling compactions & flushes 2024-11-24T11:15:19,490 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:19,491 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:19,491 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. after waiting 0 ms 2024-11-24T11:15:19,491 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:19,491 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing d57999a7f23de1f386379af6f2171765 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T11:15:19,492 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=A 2024-11-24T11:15:19,492 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:19,492 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=B 2024-11-24T11:15:19,492 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:19,492 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57999a7f23de1f386379af6f2171765, store=C 2024-11-24T11:15:19,492 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:19,502 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124385ac8c252704b9ab429a2ca88d60f2d_d57999a7f23de1f386379af6f2171765 is 50, key is test_row_0/A:col10/1732446913084/Put/seqid=0 2024-11-24T11:15:19,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742065_1241 (size=12454) 2024-11-24T11:15:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T11:15:19,909 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:19,922 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124385ac8c252704b9ab429a2ca88d60f2d_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124385ac8c252704b9ab429a2ca88d60f2d_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:19,924 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/0734ee9e264f41ff97260d7d24ed6cee, store: [table=TestAcidGuarantees family=A region=d57999a7f23de1f386379af6f2171765] 2024-11-24T11:15:19,924 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/0734ee9e264f41ff97260d7d24ed6cee is 175, key is test_row_0/A:col10/1732446913084/Put/seqid=0 2024-11-24T11:15:19,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742066_1242 (size=31255) 2024-11-24T11:15:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T11:15:20,330 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=558, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/0734ee9e264f41ff97260d7d24ed6cee 2024-11-24T11:15:20,342 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/66a29fbf79234e5781c76452eccf4ef2 is 50, key is test_row_0/B:col10/1732446913084/Put/seqid=0 2024-11-24T11:15:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742067_1243 (size=12301) 2024-11-24T11:15:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T11:15:20,748 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/66a29fbf79234e5781c76452eccf4ef2 2024-11-24T11:15:20,760 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/4df2798a41a84b2ca62b2ebbad3443f1 is 50, key is test_row_0/C:col10/1732446913084/Put/seqid=0 2024-11-24T11:15:20,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742068_1244 (size=12301) 2024-11-24T11:15:21,166 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/4df2798a41a84b2ca62b2ebbad3443f1 2024-11-24T11:15:21,177 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/A/0734ee9e264f41ff97260d7d24ed6cee as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/0734ee9e264f41ff97260d7d24ed6cee 2024-11-24T11:15:21,183 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/0734ee9e264f41ff97260d7d24ed6cee, entries=150, sequenceid=558, filesize=30.5 K 2024-11-24T11:15:21,184 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/B/66a29fbf79234e5781c76452eccf4ef2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/66a29fbf79234e5781c76452eccf4ef2 2024-11-24T11:15:21,188 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/66a29fbf79234e5781c76452eccf4ef2, entries=150, sequenceid=558, filesize=12.0 K 2024-11-24T11:15:21,189 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/.tmp/C/4df2798a41a84b2ca62b2ebbad3443f1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4df2798a41a84b2ca62b2ebbad3443f1 2024-11-24T11:15:21,192 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4df2798a41a84b2ca62b2ebbad3443f1, entries=150, sequenceid=558, filesize=12.0 K 2024-11-24T11:15:21,193 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d57999a7f23de1f386379af6f2171765 in 1701ms, sequenceid=558, compaction requested=true 2024-11-24T11:15:21,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/3389713d1de1437dad9dfc3493f8f4a0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/2f02d88c6ca2454588b459309c6d05fd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/02447491fa0a460ca686fba32b3eda3d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c5253ba5a4ea470aa99e9e470254d1a5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/8a0b10ca8e57439297bb07458a95c59e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/67143247cdf34bd692937efe87c2e669, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c58727ef24f14dbabfca6bf080a884db, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/e52c906e30d842b981f223ee7b2cd99b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d5da3d59781445fcbc7837a5108a9405, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d124c15f7beb4822b5176c58261f6572, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/07d251fa6f2e4c0d88f94f0462455468, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/4c733a14bb9b4fdd964e24ae42ff413c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9e60ec57d19045fe9fc794a399e4b2b0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9638c88548ea4aefa206cdaaab01cb67, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fe873bd7539a4614a9926bc63dad1c95, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/46e928bb8aad4730bc76ee5fdad6f450, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/88d9ab7efbc64c3f90d89f1358afa482, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/43e52eaf79164a98b8c1ac91efa0b478, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/df20b3efe3364069b9ba40d85baaba01, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/93283475c3e84c03a2088147eb7a6a3a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/84fb5aa17eca4993babc308e3de00237, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/00315eafb19c4898869d14b7a83034b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/6e9122b162764977a6068f1c5388658e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/1ba202e060cf40f0a1672b1691849a0e] to archive 2024-11-24T11:15:21,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:15:21,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/3389713d1de1437dad9dfc3493f8f4a0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/3389713d1de1437dad9dfc3493f8f4a0 2024-11-24T11:15:21,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/2f02d88c6ca2454588b459309c6d05fd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/2f02d88c6ca2454588b459309c6d05fd 2024-11-24T11:15:21,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/02447491fa0a460ca686fba32b3eda3d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/02447491fa0a460ca686fba32b3eda3d 2024-11-24T11:15:21,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c5253ba5a4ea470aa99e9e470254d1a5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c5253ba5a4ea470aa99e9e470254d1a5 2024-11-24T11:15:21,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/8a0b10ca8e57439297bb07458a95c59e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/8a0b10ca8e57439297bb07458a95c59e 2024-11-24T11:15:21,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/67143247cdf34bd692937efe87c2e669 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/67143247cdf34bd692937efe87c2e669 2024-11-24T11:15:21,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c58727ef24f14dbabfca6bf080a884db to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/c58727ef24f14dbabfca6bf080a884db 2024-11-24T11:15:21,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/e52c906e30d842b981f223ee7b2cd99b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/e52c906e30d842b981f223ee7b2cd99b 2024-11-24T11:15:21,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d5da3d59781445fcbc7837a5108a9405 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d5da3d59781445fcbc7837a5108a9405 2024-11-24T11:15:21,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d124c15f7beb4822b5176c58261f6572 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/d124c15f7beb4822b5176c58261f6572 2024-11-24T11:15:21,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/07d251fa6f2e4c0d88f94f0462455468 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/07d251fa6f2e4c0d88f94f0462455468 2024-11-24T11:15:21,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/4c733a14bb9b4fdd964e24ae42ff413c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/4c733a14bb9b4fdd964e24ae42ff413c 2024-11-24T11:15:21,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9e60ec57d19045fe9fc794a399e4b2b0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9e60ec57d19045fe9fc794a399e4b2b0 2024-11-24T11:15:21,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9638c88548ea4aefa206cdaaab01cb67 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/9638c88548ea4aefa206cdaaab01cb67 2024-11-24T11:15:21,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fe873bd7539a4614a9926bc63dad1c95 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fe873bd7539a4614a9926bc63dad1c95 2024-11-24T11:15:21,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/46e928bb8aad4730bc76ee5fdad6f450 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/46e928bb8aad4730bc76ee5fdad6f450 2024-11-24T11:15:21,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/88d9ab7efbc64c3f90d89f1358afa482 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/88d9ab7efbc64c3f90d89f1358afa482 2024-11-24T11:15:21,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/43e52eaf79164a98b8c1ac91efa0b478 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/43e52eaf79164a98b8c1ac91efa0b478 2024-11-24T11:15:21,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/df20b3efe3364069b9ba40d85baaba01 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/df20b3efe3364069b9ba40d85baaba01 2024-11-24T11:15:21,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/93283475c3e84c03a2088147eb7a6a3a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/93283475c3e84c03a2088147eb7a6a3a 2024-11-24T11:15:21,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/84fb5aa17eca4993babc308e3de00237 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/84fb5aa17eca4993babc308e3de00237 2024-11-24T11:15:21,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/00315eafb19c4898869d14b7a83034b6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/00315eafb19c4898869d14b7a83034b6 2024-11-24T11:15:21,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/6e9122b162764977a6068f1c5388658e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/6e9122b162764977a6068f1c5388658e 2024-11-24T11:15:21,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/1ba202e060cf40f0a1672b1691849a0e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/1ba202e060cf40f0a1672b1691849a0e 2024-11-24T11:15:21,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/3bf99ed6a01c4d75ac4c96788d1193c0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8173bd30bf2f4676a5c2194868c1233b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a113288301b24d3780ced566fcbc0bb9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/9357d73cda044dd4b1293369bb997b74, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/19fd1458448942aab183428d55492932, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/4bb638dc49c44665abc729c4b0201c56, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/85e5beec56a74115b79d87280cb6a92e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/cc69b13bd45243758b869c33c12cf9ef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/381ee5163aff467b9f33c868b2901849, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/33a6a9573a934e4eb02ad3a410abb0b0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/e63c6b4da78d41c2922a96f3c181a7f0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/26b9f8314247476db930d7822e9246ef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ed45e2a576a74ab6b2d033bfd96d83e6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/da2c0f31462e426483d8e1b76db5f18e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/f1c47d514a6c4f33a4e2d1eaa8ee4689, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/dcd1f451746747a2a3e26767147bca22, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a946d42b7b8545cb91bcda2e52432fb3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8c27f023bdb045f18988008bef5fb367, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/d3150ce9d0cf4b4aa238b0e4c141ba25, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b95b533d74ba4d9b98bff6af169a714d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b7f2988d66e64755840a88bc9fd879b7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ee1482283dac4d52a34d3d296f40b860, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/03253ec4a6b94fb2843c24855ffac2d3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/1512473395d4460cb60844435fb3d537] to archive 2024-11-24T11:15:21,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:15:21,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/3bf99ed6a01c4d75ac4c96788d1193c0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/3bf99ed6a01c4d75ac4c96788d1193c0 2024-11-24T11:15:21,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8173bd30bf2f4676a5c2194868c1233b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8173bd30bf2f4676a5c2194868c1233b 2024-11-24T11:15:21,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a113288301b24d3780ced566fcbc0bb9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a113288301b24d3780ced566fcbc0bb9 2024-11-24T11:15:21,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/9357d73cda044dd4b1293369bb997b74 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/9357d73cda044dd4b1293369bb997b74 2024-11-24T11:15:21,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/19fd1458448942aab183428d55492932 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/19fd1458448942aab183428d55492932 2024-11-24T11:15:21,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/4bb638dc49c44665abc729c4b0201c56 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/4bb638dc49c44665abc729c4b0201c56 2024-11-24T11:15:21,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/85e5beec56a74115b79d87280cb6a92e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/85e5beec56a74115b79d87280cb6a92e 2024-11-24T11:15:21,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/cc69b13bd45243758b869c33c12cf9ef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/cc69b13bd45243758b869c33c12cf9ef 2024-11-24T11:15:21,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/381ee5163aff467b9f33c868b2901849 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/381ee5163aff467b9f33c868b2901849 2024-11-24T11:15:21,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/33a6a9573a934e4eb02ad3a410abb0b0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/33a6a9573a934e4eb02ad3a410abb0b0 2024-11-24T11:15:21,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/e63c6b4da78d41c2922a96f3c181a7f0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/e63c6b4da78d41c2922a96f3c181a7f0 2024-11-24T11:15:21,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/26b9f8314247476db930d7822e9246ef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/26b9f8314247476db930d7822e9246ef 2024-11-24T11:15:21,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ed45e2a576a74ab6b2d033bfd96d83e6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ed45e2a576a74ab6b2d033bfd96d83e6 2024-11-24T11:15:21,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/da2c0f31462e426483d8e1b76db5f18e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/da2c0f31462e426483d8e1b76db5f18e 2024-11-24T11:15:21,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/f1c47d514a6c4f33a4e2d1eaa8ee4689 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/f1c47d514a6c4f33a4e2d1eaa8ee4689 2024-11-24T11:15:21,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/dcd1f451746747a2a3e26767147bca22 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/dcd1f451746747a2a3e26767147bca22 2024-11-24T11:15:21,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a946d42b7b8545cb91bcda2e52432fb3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/a946d42b7b8545cb91bcda2e52432fb3 2024-11-24T11:15:21,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8c27f023bdb045f18988008bef5fb367 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/8c27f023bdb045f18988008bef5fb367 2024-11-24T11:15:21,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/d3150ce9d0cf4b4aa238b0e4c141ba25 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/d3150ce9d0cf4b4aa238b0e4c141ba25 2024-11-24T11:15:21,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b95b533d74ba4d9b98bff6af169a714d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b95b533d74ba4d9b98bff6af169a714d 2024-11-24T11:15:21,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b7f2988d66e64755840a88bc9fd879b7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/b7f2988d66e64755840a88bc9fd879b7 2024-11-24T11:15:21,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ee1482283dac4d52a34d3d296f40b860 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/ee1482283dac4d52a34d3d296f40b860 2024-11-24T11:15:21,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/03253ec4a6b94fb2843c24855ffac2d3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/03253ec4a6b94fb2843c24855ffac2d3 2024-11-24T11:15:21,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/1512473395d4460cb60844435fb3d537 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/1512473395d4460cb60844435fb3d537 2024-11-24T11:15:21,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/834129da4b204a98a06d6d4d0b8ff3b8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4cbf45dac95943a1a8f6e683709075d0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a948109258c042308d14d53b12b0e5fa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/1796f18bb0d543e69f98945bd9a857ce, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/fa824bb8652b4eeea5cf8096e6b2e1aa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/ae4f84a923d34debbcff7cd996723c38, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a216ad8e53fb4c56ae57f46c18bb128a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/8613bede92344c49b19c6c081883ca9d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/6ec2888873e9449484f270492b8ec24b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/5aade8836c7141e28cd1401470572eff, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d92c0dbb0c0496f9228f73d29f962c0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/31c24053378e4df995b51f03dbee4e92, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/e94d4dcad50e4a4e946ea2019f338cce, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/c89b963ed9a6475484f1e2b403fc029a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a0d86f5012d9448c8aff01c9b9fe2846, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/303ec08f14c7429a9d8923fa55b7fa4f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/21dc84d14ba94ed0b9f6fc583cf204d5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/42bed193696f4d9898234c5392e95985, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2cc55b3939014466aab72f970f7cacce, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/72b4c8408ebc40caa96036a1f4b8613d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2e95d28b19fc4a98a271fbf2a3616527, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/59858aa16faa4fdabc44a8ab4e1ae31d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/04842d44ce724012bc99440acb8f765f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d707567d48c42a2ba06872e820114ef] to archive 2024-11-24T11:15:21,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:15:21,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/834129da4b204a98a06d6d4d0b8ff3b8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/834129da4b204a98a06d6d4d0b8ff3b8 2024-11-24T11:15:21,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4cbf45dac95943a1a8f6e683709075d0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4cbf45dac95943a1a8f6e683709075d0 2024-11-24T11:15:21,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a948109258c042308d14d53b12b0e5fa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a948109258c042308d14d53b12b0e5fa 2024-11-24T11:15:21,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/1796f18bb0d543e69f98945bd9a857ce to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/1796f18bb0d543e69f98945bd9a857ce 2024-11-24T11:15:21,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/fa824bb8652b4eeea5cf8096e6b2e1aa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/fa824bb8652b4eeea5cf8096e6b2e1aa 2024-11-24T11:15:21,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/ae4f84a923d34debbcff7cd996723c38 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/ae4f84a923d34debbcff7cd996723c38 2024-11-24T11:15:21,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a216ad8e53fb4c56ae57f46c18bb128a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a216ad8e53fb4c56ae57f46c18bb128a 2024-11-24T11:15:21,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/8613bede92344c49b19c6c081883ca9d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/8613bede92344c49b19c6c081883ca9d 2024-11-24T11:15:21,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/6ec2888873e9449484f270492b8ec24b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/6ec2888873e9449484f270492b8ec24b 2024-11-24T11:15:21,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/5aade8836c7141e28cd1401470572eff to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/5aade8836c7141e28cd1401470572eff 2024-11-24T11:15:21,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d92c0dbb0c0496f9228f73d29f962c0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d92c0dbb0c0496f9228f73d29f962c0 2024-11-24T11:15:21,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/31c24053378e4df995b51f03dbee4e92 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/31c24053378e4df995b51f03dbee4e92 2024-11-24T11:15:21,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/e94d4dcad50e4a4e946ea2019f338cce to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/e94d4dcad50e4a4e946ea2019f338cce 2024-11-24T11:15:21,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/c89b963ed9a6475484f1e2b403fc029a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/c89b963ed9a6475484f1e2b403fc029a 2024-11-24T11:15:21,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a0d86f5012d9448c8aff01c9b9fe2846 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/a0d86f5012d9448c8aff01c9b9fe2846 2024-11-24T11:15:21,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/303ec08f14c7429a9d8923fa55b7fa4f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/303ec08f14c7429a9d8923fa55b7fa4f 2024-11-24T11:15:21,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/21dc84d14ba94ed0b9f6fc583cf204d5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/21dc84d14ba94ed0b9f6fc583cf204d5 2024-11-24T11:15:21,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/42bed193696f4d9898234c5392e95985 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/42bed193696f4d9898234c5392e95985 2024-11-24T11:15:21,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2cc55b3939014466aab72f970f7cacce to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2cc55b3939014466aab72f970f7cacce 2024-11-24T11:15:21,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/72b4c8408ebc40caa96036a1f4b8613d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/72b4c8408ebc40caa96036a1f4b8613d 2024-11-24T11:15:21,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2e95d28b19fc4a98a271fbf2a3616527 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/2e95d28b19fc4a98a271fbf2a3616527 2024-11-24T11:15:21,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/59858aa16faa4fdabc44a8ab4e1ae31d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/59858aa16faa4fdabc44a8ab4e1ae31d 2024-11-24T11:15:21,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/04842d44ce724012bc99440acb8f765f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/04842d44ce724012bc99440acb8f765f 2024-11-24T11:15:21,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d707567d48c42a2ba06872e820114ef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/7d707567d48c42a2ba06872e820114ef 2024-11-24T11:15:21,266 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/recovered.edits/561.seqid, newMaxSeqId=561, maxSeqId=4 2024-11-24T11:15:21,267 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. 2024-11-24T11:15:21,267 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for d57999a7f23de1f386379af6f2171765: Waiting for close lock at 1732446919490Running coprocessor pre-close hooks at 1732446919490Disabling compacts and flushes for region at 1732446919490Disabling writes for close at 1732446919491 (+1 ms)Obtaining lock to block concurrent updates at 1732446919491Preparing flush snapshotting stores in d57999a7f23de1f386379af6f2171765 at 1732446919491Finished memstore snapshotting TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732446919492 (+1 ms)Flushing stores of TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765. at 1732446919494 (+2 ms)Flushing d57999a7f23de1f386379af6f2171765/A: creating writer at 1732446919494Flushing mob file d57999a7f23de1f386379af6f2171765/A: appending metadata at 1732446919502 (+8 ms)Flushing mob file d57999a7f23de1f386379af6f2171765/A: closing flushed file at 1732446919502Flushing d57999a7f23de1f386379af6f2171765/A: appending metadata at 1732446919924 (+422 ms)Flushing d57999a7f23de1f386379af6f2171765/A: closing flushed file at 1732446919924Flushing d57999a7f23de1f386379af6f2171765/B: creating writer at 1732446920339 (+415 ms)Flushing d57999a7f23de1f386379af6f2171765/B: appending metadata at 1732446920342 (+3 ms)Flushing d57999a7f23de1f386379af6f2171765/B: closing flushed file at 1732446920342Flushing d57999a7f23de1f386379af6f2171765/C: creating writer at 1732446920757 (+415 ms)Flushing d57999a7f23de1f386379af6f2171765/C: appending metadata at 1732446920760 (+3 ms)Flushing d57999a7f23de1f386379af6f2171765/C: closing flushed file at 1732446920760Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195d58a9: reopening flushed file at 1732446921176 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22b598bb: reopening flushed file at 1732446921184 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6162dbeb: reopening flushed file at 1732446921188 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d57999a7f23de1f386379af6f2171765 in 1701ms, sequenceid=558, compaction requested=true at 1732446921193 (+5 ms)Writing region close event to WAL at 1732446921263 (+70 ms)Running coprocessor post-close hooks at 1732446921267 (+4 ms)Closed at 1732446921267 2024-11-24T11:15:21,269 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,269 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=d57999a7f23de1f386379af6f2171765, regionState=CLOSED 2024-11-24T11:15:21,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:15:21,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T11:15:21,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure d57999a7f23de1f386379af6f2171765, server=f5f73e76a839,45939,1732446856176 in 1.9340 sec 2024-11-24T11:15:21,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-24T11:15:21,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57999a7f23de1f386379af6f2171765, UNASSIGN in 1.9400 sec 2024-11-24T11:15:21,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T11:15:21,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9440 sec 2024-11-24T11:15:21,277 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446921276"}]},"ts":"1732446921276"} 2024-11-24T11:15:21,278 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T11:15:21,278 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T11:15:21,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9540 sec 2024-11-24T11:15:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T11:15:21,474 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:21,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-24T11:15:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:21,478 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T11:15:21,479 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:21,485 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,487 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/recovered.edits] 2024-11-24T11:15:21,491 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/0734ee9e264f41ff97260d7d24ed6cee to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/0734ee9e264f41ff97260d7d24ed6cee 2024-11-24T11:15:21,495 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/911b8cceb0bc4a91a14591d6cef58536 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/911b8cceb0bc4a91a14591d6cef58536 2024-11-24T11:15:21,496 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/dde86a33557b4e3f8902d1f41501ba6e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/dde86a33557b4e3f8902d1f41501ba6e 2024-11-24T11:15:21,498 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fef1022958a544b6a1b810c67853a4f5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/A/fef1022958a544b6a1b810c67853a4f5 2024-11-24T11:15:21,500 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/66a29fbf79234e5781c76452eccf4ef2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/66a29fbf79234e5781c76452eccf4ef2 2024-11-24T11:15:21,502 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/7e75d7e5015f491482eb19aca1cff615 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/7e75d7e5015f491482eb19aca1cff615 2024-11-24T11:15:21,504 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/981db041e0634b368a07d4fd226eae15 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/981db041e0634b368a07d4fd226eae15 2024-11-24T11:15:21,506 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/fedb9d6a899b41ba8eb6770e3ee7978f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/B/fedb9d6a899b41ba8eb6770e3ee7978f 2024-11-24T11:15:21,509 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4df2798a41a84b2ca62b2ebbad3443f1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/4df2798a41a84b2ca62b2ebbad3443f1 2024-11-24T11:15:21,511 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/77517c4c00214aaba83d993314ac1321 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/77517c4c00214aaba83d993314ac1321 2024-11-24T11:15:21,514 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/b076301b44eb4633bc356e63f1bf2f67 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/b076301b44eb4633bc356e63f1bf2f67 2024-11-24T11:15:21,516 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/b82ae1d6f08b4ee480478c4eae669469 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/C/b82ae1d6f08b4ee480478c4eae669469 2024-11-24T11:15:21,520 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/recovered.edits/561.seqid to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765/recovered.edits/561.seqid 2024-11-24T11:15:21,521 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,521 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T11:15:21,522 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T11:15:21,523 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T11:15:21,531 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112407698ce20c524fbf8e139b5fbd3766d7_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112407698ce20c524fbf8e139b5fbd3766d7_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,534 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242838a0d2f7c64febbfd7a310b31e2211_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242838a0d2f7c64febbfd7a310b31e2211_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,536 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124385ac8c252704b9ab429a2ca88d60f2d_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124385ac8c252704b9ab429a2ca88d60f2d_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,538 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243a08b34c53354e0b8823edf4625e3fde_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243a08b34c53354e0b8823edf4625e3fde_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,541 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124496e6f260bd94ccda8eed93916dbe54e_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124496e6f260bd94ccda8eed93916dbe54e_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,545 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ac49b4216fa42e086ea28a8f2a6d56e_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ac49b4216fa42e086ea28a8f2a6d56e_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,547 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b5d8a09ab43419f815a3af86c3f6804_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b5d8a09ab43419f815a3af86c3f6804_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,549 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244d0b7949ef3b458fa2490208b94f6194_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244d0b7949ef3b458fa2490208b94f6194_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,551 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246422b89bfaa04049a217b577d19765ae_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246422b89bfaa04049a217b577d19765ae_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,553 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247073a5f0b61a493681eb203a062b1c0b_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247073a5f0b61a493681eb203a062b1c0b_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,554 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247ed03fc2d9674da0bf361dc1655704aa_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247ed03fc2d9674da0bf361dc1655704aa_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,555 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ef8cb7e115f45d9bd4f5f195eaff2e8_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ef8cb7e115f45d9bd4f5f195eaff2e8_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124905e4e1d8eaf4146aebb0dd030589108_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124905e4e1d8eaf4146aebb0dd030589108_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,558 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112490b3d68bfcdb4f1094e21e7232b8785b_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112490b3d68bfcdb4f1094e21e7232b8785b_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,560 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124921206f98418414a8299b3e9b73f37db_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124921206f98418414a8299b3e9b73f37db_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249930a06b366e4ab59b9321eaf39586bd_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249930a06b366e4ab59b9321eaf39586bd_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249971adacfb10425ebb18985c543af4d4_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249971adacfb10425ebb18985c543af4d4_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,566 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b1a3b18428cb44bfabde81037119f5dc_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b1a3b18428cb44bfabde81037119f5dc_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,567 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bc119096b0054a5f916a1d7a2c581ed8_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bc119096b0054a5f916a1d7a2c581ed8_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,569 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd95716b0c384526aa5b6f8124c0d200_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd95716b0c384526aa5b6f8124c0d200_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,570 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fd3e074e43e44985bd2175c14e50c43d_d57999a7f23de1f386379af6f2171765 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fd3e074e43e44985bd2175c14e50c43d_d57999a7f23de1f386379af6f2171765 2024-11-24T11:15:21,572 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T11:15:21,577 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:21,580 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T11:15:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T11:15:21,583 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T11:15:21,585 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:21,585 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T11:15:21,585 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732446921585"}]},"ts":"9223372036854775807"} 2024-11-24T11:15:21,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T11:15:21,589 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d57999a7f23de1f386379af6f2171765, NAME => 'TestAcidGuarantees,,1732446887275.d57999a7f23de1f386379af6f2171765.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T11:15:21,589 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T11:15:21,590 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732446921589"}]},"ts":"9223372036854775807"} 2024-11-24T11:15:21,593 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T11:15:21,595 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:21,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 119 msec 2024-11-24T11:15:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T11:15:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T11:15:21,794 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:21,801 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-452383080_22 at /127.0.0.1:44082 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=394 (was 425), ProcessCount=11 (was 11), AvailableMemoryMB=5993 (was 6235) 2024-11-24T11:15:21,811 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=394, ProcessCount=11, AvailableMemoryMB=5993 2024-11-24T11:15:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:15:21,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:15:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:21,815 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T11:15:21,815 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:21,815 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-24T11:15:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T11:15:21,816 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T11:15:21,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742069_1245 (size=1061) 2024-11-24T11:15:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T11:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T11:15:22,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:15:22,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742070_1246 (size=53) 2024-11-24T11:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T11:15:22,636 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:15:22,636 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 76c783a6d487b45ef515b468f5ea8d41, disabling compactions & flushes 2024-11-24T11:15:22,636 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,637 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,637 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. after waiting 0 ms 2024-11-24T11:15:22,637 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,637 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,637 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76c783a6d487b45ef515b468f5ea8d41: Waiting for close lock at 1732446922636Disabling compacts and flushes for region at 1732446922636Disabling writes for close at 1732446922637 (+1 ms)Writing region close event to WAL at 1732446922637Closed at 1732446922637 2024-11-24T11:15:22,640 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T11:15:22,640 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732446922640"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732446922640"}]},"ts":"1732446922640"} 2024-11-24T11:15:22,642 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T11:15:22,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T11:15:22,643 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446922643"}]},"ts":"1732446922643"} 2024-11-24T11:15:22,645 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T11:15:22,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, ASSIGN}] 2024-11-24T11:15:22,646 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, ASSIGN 2024-11-24T11:15:22,647 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, ASSIGN; state=OFFLINE, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=false 2024-11-24T11:15:22,799 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=76c783a6d487b45ef515b468f5ea8d41, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:22,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, ASSIGN because future has completed 2024-11-24T11:15:22,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T11:15:22,963 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,963 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:15:22,964 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,964 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:15:22,964 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,964 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,966 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,968 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:22,968 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c783a6d487b45ef515b468f5ea8d41 columnFamilyName A 2024-11-24T11:15:22,968 DEBUG [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:22,969 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.HStore(327): Store=76c783a6d487b45ef515b468f5ea8d41/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:22,969 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,971 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:22,971 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c783a6d487b45ef515b468f5ea8d41 columnFamilyName B 2024-11-24T11:15:22,971 DEBUG [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:22,972 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.HStore(327): Store=76c783a6d487b45ef515b468f5ea8d41/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:22,972 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,974 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:22,974 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c783a6d487b45ef515b468f5ea8d41 columnFamilyName C 2024-11-24T11:15:22,975 DEBUG [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:22,975 INFO [StoreOpener-76c783a6d487b45ef515b468f5ea8d41-1 {}] regionserver.HStore(327): Store=76c783a6d487b45ef515b468f5ea8d41/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:22,975 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,975 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,976 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,977 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,978 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,978 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,979 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:15:22,980 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,982 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:15:22,983 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 76c783a6d487b45ef515b468f5ea8d41; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60681575, jitterRate=-0.09577406942844391}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:15:22,983 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:22,984 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 76c783a6d487b45ef515b468f5ea8d41: Running coprocessor pre-open hook at 1732446922964Writing region info on filesystem at 1732446922964Initializing all the Stores at 1732446922965 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446922965Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446922966 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446922966Cleaning up temporary data from old regions at 1732446922978 (+12 ms)Running coprocessor post-open hooks at 1732446922983 (+5 ms)Region opened successfully at 1732446922984 (+1 ms) 2024-11-24T11:15:22,984 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., pid=59, masterSystemTime=1732446922958 2024-11-24T11:15:22,986 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,986 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:22,986 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=76c783a6d487b45ef515b468f5ea8d41, regionState=OPEN, openSeqNum=2, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:22,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:15:22,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-24T11:15:22,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 in 185 msec 2024-11-24T11:15:22,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T11:15:22,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, ASSIGN in 345 msec 2024-11-24T11:15:22,992 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T11:15:22,992 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446922992"}]},"ts":"1732446922992"} 2024-11-24T11:15:22,993 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T11:15:22,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T11:15:22,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-24T11:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T11:15:23,965 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:23,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cbe010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:23,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:23,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297888a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:23,975 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c74d38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,976 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:23,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:23,980 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T11:15:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f5f73e76a839,40345,1732446855400 2024-11-24T11:15:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@519e4a3 2024-11-24T11:15:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T11:15:23,982 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T11:15:23,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1c5003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:23,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:23,985 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c47db9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:23,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f389e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cde69b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:23,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:23,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20605630, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:23,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2c41f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cf162e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:23,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:23,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22945713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:23,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419a23d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240c3b1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:23,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:23,999 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7baeb517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:24,000 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588481da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5faf75fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:24,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:24,002 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46088a28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:24,004 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a21332f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d01f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:24,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:24,006 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c15cdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:24,007 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f70301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdd3a3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:24,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:24,009 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a64f64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:24,010 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da763df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fafaf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:24,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:24,012 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ff76da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:24,013 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc68bae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f507a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:24,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:24,015 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7f469c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:24,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fa6d8a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35038a97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:24,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:24,018 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccf3b78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:24,019 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f9e97f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:24,020 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,020 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,020 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,020 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,020 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:24,020 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,021 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,021 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,021 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-24T11:15:24,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,024 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T11:15:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,025 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,025 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,028 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:24,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:15:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:24,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T11:15:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:24,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/ea70fe4fa82d49ec8dbca77f87ff8164 is 50, key is test_row_0/A:col10/1732446924033/Put/seqid=0 2024-11-24T11:15:24,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446984058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446924050, max timestamp=1732446924062Num uniques 859; 2024-11-24T11:15:24,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446924050, max timestamp=1732446924062Num uniques 859; 2024-11-24T11:15:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:24,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446984061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446924050, max timestamp=1732446924062Num uniques 859; 2024-11-24T11:15:24,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:24,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446984093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446984094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446984097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742071_1247 (size=12001) 2024-11-24T11:15:24,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/ea70fe4fa82d49ec8dbca77f87ff8164 2024-11-24T11:15:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T11:15:24,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f8f30f27b09441d3b99bddc769179ccb is 50, key is test_row_0/B:col10/1732446924033/Put/seqid=0 2024-11-24T11:15:24,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742072_1248 (size=12001) 2024-11-24T11:15:24,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446984174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446984174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T11:15:24,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:24,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446984203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446984203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446984213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T11:15:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:24,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T11:15:24,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446984383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446984383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446984413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446984413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446984424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T11:15:24,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:24,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f8f30f27b09441d3b99bddc769179ccb 2024-11-24T11:15:24,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e6571c1e720f4064a359374b4110cbfc is 50, key is test_row_0/C:col10/1732446924033/Put/seqid=0 2024-11-24T11:15:24,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742073_1249 (size=12001) 2024-11-24T11:15:24,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T11:15:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T11:15:24,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446984694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446984694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446984723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446984723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446984733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:24,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T11:15:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T11:15:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:24,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:24,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e6571c1e720f4064a359374b4110cbfc 2024-11-24T11:15:24,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/ea70fe4fa82d49ec8dbca77f87ff8164 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/ea70fe4fa82d49ec8dbca77f87ff8164 2024-11-24T11:15:24,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/ea70fe4fa82d49ec8dbca77f87ff8164, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T11:15:24,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f8f30f27b09441d3b99bddc769179ccb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f8f30f27b09441d3b99bddc769179ccb 2024-11-24T11:15:24,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f8f30f27b09441d3b99bddc769179ccb, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T11:15:24,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e6571c1e720f4064a359374b4110cbfc as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e6571c1e720f4064a359374b4110cbfc 2024-11-24T11:15:24,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e6571c1e720f4064a359374b4110cbfc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T11:15:24,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76c783a6d487b45ef515b468f5ea8d41 in 958ms, sequenceid=12, compaction requested=false 2024-11-24T11:15:24,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:25,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T11:15:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:25,100 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T11:15:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:25,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/f1f6edaa1b0d4ff0a817e7783d682fe9 is 50, key is test_row_0/A:col10/1732446924092/Put/seqid=0 2024-11-24T11:15:25,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742074_1250 (size=12001) 2024-11-24T11:15:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T11:15:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:25,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446985222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446985223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446985234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446985234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446985243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446985334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446985334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,509 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/f1f6edaa1b0d4ff0a817e7783d682fe9 2024-11-24T11:15:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/2ded6648ced1405a9256c686294ecfba is 50, key is test_row_0/B:col10/1732446924092/Put/seqid=0 2024-11-24T11:15:25,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742075_1251 (size=12001) 2024-11-24T11:15:25,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446985543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446985544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446985853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446985853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:25,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/2ded6648ced1405a9256c686294ecfba 2024-11-24T11:15:25,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/8a0e89bda1ee4a2bba86d536cfcf80ab is 50, key is test_row_0/C:col10/1732446924092/Put/seqid=0 2024-11-24T11:15:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742076_1252 (size=12001) 2024-11-24T11:15:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T11:15:26,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446986244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:26,247 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:26,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446986254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:26,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446986254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:26,257 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:26,257 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:26,346 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/8a0e89bda1ee4a2bba86d536cfcf80ab 2024-11-24T11:15:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446986363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446986363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/f1f6edaa1b0d4ff0a817e7783d682fe9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/f1f6edaa1b0d4ff0a817e7783d682fe9 2024-11-24T11:15:26,413 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/f1f6edaa1b0d4ff0a817e7783d682fe9, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T11:15:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/2ded6648ced1405a9256c686294ecfba as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2ded6648ced1405a9256c686294ecfba 2024-11-24T11:15:26,419 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2ded6648ced1405a9256c686294ecfba, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T11:15:26,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/8a0e89bda1ee4a2bba86d536cfcf80ab as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8a0e89bda1ee4a2bba86d536cfcf80ab 2024-11-24T11:15:26,423 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8a0e89bda1ee4a2bba86d536cfcf80ab, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T11:15:26,424 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 76c783a6d487b45ef515b468f5ea8d41 in 1324ms, sequenceid=47, compaction requested=false 2024-11-24T11:15:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-24T11:15:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-24T11:15:26,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T11:15:26,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3980 sec 2024-11-24T11:15:26,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.4090 sec 2024-11-24T11:15:26,794 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T11:15:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:27,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:15:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:27,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/92b331576e0747bd96bab332f55343c4 is 50, key is test_row_0/A:col10/1732446927376/Put/seqid=0 2024-11-24T11:15:27,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742077_1253 (size=14341) 2024-11-24T11:15:27,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/92b331576e0747bd96bab332f55343c4 2024-11-24T11:15:27,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/fed4164292ac4c76bf62036b798cbc01 is 50, key is test_row_0/B:col10/1732446927376/Put/seqid=0 2024-11-24T11:15:27,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742078_1254 (size=12001) 2024-11-24T11:15:27,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446987414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:27,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446987415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:27,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446987523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:27,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446987523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446987734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446987734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:27,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/fed4164292ac4c76bf62036b798cbc01 2024-11-24T11:15:27,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/a15af08f970e4af4ab762f7fe5ce363b is 50, key is test_row_0/C:col10/1732446927376/Put/seqid=0 2024-11-24T11:15:27,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742079_1255 (size=12001) 2024-11-24T11:15:28,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446988043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446988043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T11:15:28,194 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:28,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-24T11:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T11:15:28,196 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:28,196 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:28,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:28,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/a15af08f970e4af4ab762f7fe5ce363b 2024-11-24T11:15:28,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/92b331576e0747bd96bab332f55343c4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/92b331576e0747bd96bab332f55343c4 2024-11-24T11:15:28,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/92b331576e0747bd96bab332f55343c4, entries=200, sequenceid=60, filesize=14.0 K 2024-11-24T11:15:28,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/fed4164292ac4c76bf62036b798cbc01 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/fed4164292ac4c76bf62036b798cbc01 2024-11-24T11:15:28,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/fed4164292ac4c76bf62036b798cbc01, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T11:15:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/a15af08f970e4af4ab762f7fe5ce363b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a15af08f970e4af4ab762f7fe5ce363b 2024-11-24T11:15:28,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a15af08f970e4af4ab762f7fe5ce363b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T11:15:28,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 76c783a6d487b45ef515b468f5ea8d41 in 869ms, sequenceid=60, compaction requested=true 2024-11-24T11:15:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:28,246 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:28,246 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:28,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/A is initiating minor compaction (all files) 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/B is initiating minor compaction (all files) 2024-11-24T11:15:28,247 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/A in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,247 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/B in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,247 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f8f30f27b09441d3b99bddc769179ccb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2ded6648ced1405a9256c686294ecfba, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/fed4164292ac4c76bf62036b798cbc01] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=35.2 K 2024-11-24T11:15:28,247 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/ea70fe4fa82d49ec8dbca77f87ff8164, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/f1f6edaa1b0d4ff0a817e7783d682fe9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/92b331576e0747bd96bab332f55343c4] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=37.4 K 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea70fe4fa82d49ec8dbca77f87ff8164, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446924033 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f8f30f27b09441d3b99bddc769179ccb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446924033 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1f6edaa1b0d4ff0a817e7783d682fe9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732446924085 2024-11-24T11:15:28,247 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ded6648ced1405a9256c686294ecfba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732446924085 2024-11-24T11:15:28,248 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b331576e0747bd96bab332f55343c4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732446925210 2024-11-24T11:15:28,248 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting fed4164292ac4c76bf62036b798cbc01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732446925221 2024-11-24T11:15:28,253 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:28,254 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/389b511f3ec34c3aa6fd1ff9926cd065 is 50, key is test_row_0/A:col10/1732446927376/Put/seqid=0 2024-11-24T11:15:28,254 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#B#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:28,254 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/c7087ce4a8c646d1bb0f0dd5f5c3e082 is 50, key is test_row_0/B:col10/1732446927376/Put/seqid=0 2024-11-24T11:15:28,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742081_1257 (size=12104) 2024-11-24T11:15:28,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742080_1256 (size=12104) 2024-11-24T11:15:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:28,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T11:15:28,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:28,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:28,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:28,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:28,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:28,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:28,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d2bf3211dc0a4ad8928c764cc1fd85d9 is 50, key is test_row_0/A:col10/1732446928267/Put/seqid=0 2024-11-24T11:15:28,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742082_1258 (size=14341) 2024-11-24T11:15:28,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446988281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446988281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446988284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T11:15:28,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446988394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446988394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446988394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T11:15:28,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446988553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446988553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446988604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446988604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446988604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:28,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:28,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,663 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/c7087ce4a8c646d1bb0f0dd5f5c3e082 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/c7087ce4a8c646d1bb0f0dd5f5c3e082 2024-11-24T11:15:28,664 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/389b511f3ec34c3aa6fd1ff9926cd065 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/389b511f3ec34c3aa6fd1ff9926cd065 2024-11-24T11:15:28,667 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/B of 76c783a6d487b45ef515b468f5ea8d41 into c7087ce4a8c646d1bb0f0dd5f5c3e082(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:28,667 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:28,668 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/B, priority=13, startTime=1732446928246; duration=0sec 2024-11-24T11:15:28,668 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:28,668 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:B 2024-11-24T11:15:28,668 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:28,669 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:28,669 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/C is initiating minor compaction (all files) 2024-11-24T11:15:28,669 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/A of 76c783a6d487b45ef515b468f5ea8d41 into 389b511f3ec34c3aa6fd1ff9926cd065(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:28,669 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/C in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,669 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:28,669 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/A, priority=13, startTime=1732446928246; duration=0sec 2024-11-24T11:15:28,669 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e6571c1e720f4064a359374b4110cbfc, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8a0e89bda1ee4a2bba86d536cfcf80ab, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a15af08f970e4af4ab762f7fe5ce363b] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=35.2 K 2024-11-24T11:15:28,670 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:28,670 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:A 2024-11-24T11:15:28,670 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e6571c1e720f4064a359374b4110cbfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446924033 2024-11-24T11:15:28,671 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0e89bda1ee4a2bba86d536cfcf80ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732446924085 2024-11-24T11:15:28,671 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a15af08f970e4af4ab762f7fe5ce363b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732446925221 2024-11-24T11:15:28,678 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#C#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:28,679 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/70a764736f0e45a3908613fe09fb0c31 is 50, key is test_row_0/C:col10/1732446927376/Put/seqid=0 2024-11-24T11:15:28,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d2bf3211dc0a4ad8928c764cc1fd85d9 2024-11-24T11:15:28,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742083_1259 (size=12104) 2024-11-24T11:15:28,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/b5c8a2b9e4184e429a8318c8117171a5 is 50, key is test_row_0/B:col10/1732446928267/Put/seqid=0 2024-11-24T11:15:28,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742084_1260 (size=12001) 2024-11-24T11:15:28,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T11:15:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446988914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446988914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446988914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:28,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:28,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:28,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:28,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,103 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/70a764736f0e45a3908613fe09fb0c31 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/70a764736f0e45a3908613fe09fb0c31 2024-11-24T11:15:29,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/b5c8a2b9e4184e429a8318c8117171a5 2024-11-24T11:15:29,107 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/C of 76c783a6d487b45ef515b468f5ea8d41 into 70a764736f0e45a3908613fe09fb0c31(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:29,107 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:29,107 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/C, priority=13, startTime=1732446928246; duration=0sec 2024-11-24T11:15:29,107 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:29,107 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:C 2024-11-24T11:15:29,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e1c50b9c8d6542b0add6c4c3ddbe0a97 is 50, key is test_row_0/C:col10/1732446928267/Put/seqid=0 2024-11-24T11:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742085_1261 (size=12001) 2024-11-24T11:15:29,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T11:15:29,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:29,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:29,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446989424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446989424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446989443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e1c50b9c8d6542b0add6c4c3ddbe0a97 2024-11-24T11:15:29,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d2bf3211dc0a4ad8928c764cc1fd85d9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d2bf3211dc0a4ad8928c764cc1fd85d9 2024-11-24T11:15:29,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d2bf3211dc0a4ad8928c764cc1fd85d9, entries=200, sequenceid=81, filesize=14.0 K 2024-11-24T11:15:29,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/b5c8a2b9e4184e429a8318c8117171a5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/b5c8a2b9e4184e429a8318c8117171a5 2024-11-24T11:15:29,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/b5c8a2b9e4184e429a8318c8117171a5, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T11:15:29,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e1c50b9c8d6542b0add6c4c3ddbe0a97 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e1c50b9c8d6542b0add6c4c3ddbe0a97 2024-11-24T11:15:29,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e1c50b9c8d6542b0add6c4c3ddbe0a97, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T11:15:29,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 76c783a6d487b45ef515b468f5ea8d41 in 1273ms, sequenceid=81, compaction requested=false 2024-11-24T11:15:29,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:29,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:15:29,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:29,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:29,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:29,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:29,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:29,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:29,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/6d7f1a7749964b9d87bd2b38b450ae5d is 50, key is test_row_0/A:col10/1732446929564/Put/seqid=0 2024-11-24T11:15:29,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742086_1262 (size=14341) 2024-11-24T11:15:29,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/6d7f1a7749964b9d87bd2b38b450ae5d 2024-11-24T11:15:29,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/7bc72593100f4da0a9aa681b9f40a7f4 is 50, key is test_row_0/B:col10/1732446929564/Put/seqid=0 2024-11-24T11:15:29,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742087_1263 (size=12001) 2024-11-24T11:15:29,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446989609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446989611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:29,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446989724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446989724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:29,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:29,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:29,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:29,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446989934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446989934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/7bc72593100f4da0a9aa681b9f40a7f4 2024-11-24T11:15:30,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/72587eb13f8a49b1aff268b59737d42f is 50, key is test_row_0/C:col10/1732446929564/Put/seqid=0 2024-11-24T11:15:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742088_1264 (size=12001) 2024-11-24T11:15:30,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446990243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446990243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T11:15:30,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/72587eb13f8a49b1aff268b59737d42f 2024-11-24T11:15:30,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/6d7f1a7749964b9d87bd2b38b450ae5d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6d7f1a7749964b9d87bd2b38b450ae5d 2024-11-24T11:15:30,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6d7f1a7749964b9d87bd2b38b450ae5d, entries=200, sequenceid=96, filesize=14.0 K 2024-11-24T11:15:30,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/7bc72593100f4da0a9aa681b9f40a7f4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/7bc72593100f4da0a9aa681b9f40a7f4 2024-11-24T11:15:30,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/7bc72593100f4da0a9aa681b9f40a7f4, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T11:15:30,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/72587eb13f8a49b1aff268b59737d42f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/72587eb13f8a49b1aff268b59737d42f 2024-11-24T11:15:30,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/72587eb13f8a49b1aff268b59737d42f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T11:15:30,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 76c783a6d487b45ef515b468f5ea8d41 in 862ms, sequenceid=96, compaction requested=true 2024-11-24T11:15:30,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:30,428 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:30,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:30,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:30,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:30,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:30,428 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:30,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:30,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:30,429 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:30,429 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/A is initiating minor compaction (all files) 2024-11-24T11:15:30,429 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/A in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,429 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/389b511f3ec34c3aa6fd1ff9926cd065, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d2bf3211dc0a4ad8928c764cc1fd85d9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6d7f1a7749964b9d87bd2b38b450ae5d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=39.8 K 2024-11-24T11:15:30,429 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:30,429 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/B is initiating minor compaction (all files) 2024-11-24T11:15:30,429 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389b511f3ec34c3aa6fd1ff9926cd065, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732446925221 2024-11-24T11:15:30,429 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/B in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,429 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/c7087ce4a8c646d1bb0f0dd5f5c3e082, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/b5c8a2b9e4184e429a8318c8117171a5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/7bc72593100f4da0a9aa681b9f40a7f4] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=35.3 K 2024-11-24T11:15:30,429 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2bf3211dc0a4ad8928c764cc1fd85d9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732446927413 2024-11-24T11:15:30,430 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c7087ce4a8c646d1bb0f0dd5f5c3e082, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732446925221 2024-11-24T11:15:30,430 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7f1a7749964b9d87bd2b38b450ae5d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446928280 2024-11-24T11:15:30,430 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b5c8a2b9e4184e429a8318c8117171a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732446927414 2024-11-24T11:15:30,430 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bc72593100f4da0a9aa681b9f40a7f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446928280 2024-11-24T11:15:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:30,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T11:15:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:30,438 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:30,439 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/d20af67afff94dfbaea9534ebb952a28 is 50, key is test_row_0/B:col10/1732446929564/Put/seqid=0 2024-11-24T11:15:30,446 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:30,446 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d5acb9bf927444d7afa1c7812e9889e2 is 50, key is test_row_0/A:col10/1732446929564/Put/seqid=0 2024-11-24T11:15:30,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742089_1265 (size=12207) 2024-11-24T11:15:30,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/a755ee63e78341b2a292bdd292bb15e8 is 50, key is test_row_0/A:col10/1732446929596/Put/seqid=0 2024-11-24T11:15:30,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742090_1266 (size=12207) 2024-11-24T11:15:30,457 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/d20af67afff94dfbaea9534ebb952a28 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/d20af67afff94dfbaea9534ebb952a28 2024-11-24T11:15:30,463 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d5acb9bf927444d7afa1c7812e9889e2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d5acb9bf927444d7afa1c7812e9889e2 2024-11-24T11:15:30,464 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/B of 76c783a6d487b45ef515b468f5ea8d41 into d20af67afff94dfbaea9534ebb952a28(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:30,464 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:30,464 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/B, priority=13, startTime=1732446930428; duration=0sec 2024-11-24T11:15:30,464 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:30,464 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:B 2024-11-24T11:15:30,464 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:30,465 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:30,465 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/C is initiating minor compaction (all files) 2024-11-24T11:15:30,465 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/C in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,465 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/70a764736f0e45a3908613fe09fb0c31, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e1c50b9c8d6542b0add6c4c3ddbe0a97, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/72587eb13f8a49b1aff268b59737d42f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=35.3 K 2024-11-24T11:15:30,466 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a764736f0e45a3908613fe09fb0c31, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732446925221 2024-11-24T11:15:30,466 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e1c50b9c8d6542b0add6c4c3ddbe0a97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732446927414 2024-11-24T11:15:30,466 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 72587eb13f8a49b1aff268b59737d42f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446928280 2024-11-24T11:15:30,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446990461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446990461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,470 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/A of 76c783a6d487b45ef515b468f5ea8d41 into d5acb9bf927444d7afa1c7812e9889e2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:30,470 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:30,470 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/A, priority=13, startTime=1732446930427; duration=0sec 2024-11-24T11:15:30,470 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:30,470 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:A 2024-11-24T11:15:30,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742091_1267 (size=14341) 2024-11-24T11:15:30,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446990467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,478 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:30,479 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/8f7dfc7c3ea146a8a3a13c526ce1f39e is 50, key is test_row_0/C:col10/1732446929564/Put/seqid=0 2024-11-24T11:15:30,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742092_1268 (size=12207) 2024-11-24T11:15:30,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446990573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446990573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446990584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446990753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446990753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446990783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446990784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:30,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446990794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:30,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/a755ee63e78341b2a292bdd292bb15e8 2024-11-24T11:15:30,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/da3ab028553641bfbf81579e9589282d is 50, key is test_row_0/B:col10/1732446929596/Put/seqid=0 2024-11-24T11:15:30,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742093_1269 (size=12001) 2024-11-24T11:15:30,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/da3ab028553641bfbf81579e9589282d 2024-11-24T11:15:30,890 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/8f7dfc7c3ea146a8a3a13c526ce1f39e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8f7dfc7c3ea146a8a3a13c526ce1f39e 2024-11-24T11:15:30,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/49243d09315542d281b510fc08f5599d is 50, key is test_row_0/C:col10/1732446929596/Put/seqid=0 2024-11-24T11:15:30,895 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/C of 76c783a6d487b45ef515b468f5ea8d41 into 8f7dfc7c3ea146a8a3a13c526ce1f39e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:30,895 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:30,895 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/C, priority=13, startTime=1732446930428; duration=0sec 2024-11-24T11:15:30,895 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:30,895 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:C 2024-11-24T11:15:30,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742094_1270 (size=12001) 2024-11-24T11:15:30,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/49243d09315542d281b510fc08f5599d 2024-11-24T11:15:30,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/a755ee63e78341b2a292bdd292bb15e8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a755ee63e78341b2a292bdd292bb15e8 2024-11-24T11:15:30,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a755ee63e78341b2a292bdd292bb15e8, entries=200, sequenceid=117, filesize=14.0 K 2024-11-24T11:15:30,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/da3ab028553641bfbf81579e9589282d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/da3ab028553641bfbf81579e9589282d 2024-11-24T11:15:30,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/da3ab028553641bfbf81579e9589282d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T11:15:30,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/49243d09315542d281b510fc08f5599d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/49243d09315542d281b510fc08f5599d 2024-11-24T11:15:30,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/49243d09315542d281b510fc08f5599d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T11:15:30,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 76c783a6d487b45ef515b468f5ea8d41 in 476ms, sequenceid=117, compaction requested=false 2024-11-24T11:15:30,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:30,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T11:15:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:30,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T11:15:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:30,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/0d4973305911494fa1000faf75c6ad9f is 50, key is test_row_0/A:col10/1732446930464/Put/seqid=0 2024-11-24T11:15:30,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742095_1271 (size=12001) 2024-11-24T11:15:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:31,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:31,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732446931119, max timestamp=1732446931134Num uniques 859; 2024-11-24T11:15:31,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732446931119, max timestamp=1732446931134Num uniques 859; 2024-11-24T11:15:31,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:31,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:15:31,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446991158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446991158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446991159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446991274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446991274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446991274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,357 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/0d4973305911494fa1000faf75c6ad9f 2024-11-24T11:15:31,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/2e49fb6336f748c9b729e4b48e652133 is 50, key is test_row_0/B:col10/1732446930464/Put/seqid=0 2024-11-24T11:15:31,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742096_1272 (size=12001) 2024-11-24T11:15:31,427 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/2e49fb6336f748c9b729e4b48e652133 2024-11-24T11:15:31,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/c139d74ddc5244898a436e8442900108 is 50, key is test_row_0/C:col10/1732446930464/Put/seqid=0 2024-11-24T11:15:31,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742097_1273 (size=12001) 2024-11-24T11:15:31,437 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/c139d74ddc5244898a436e8442900108 2024-11-24T11:15:31,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/0d4973305911494fa1000faf75c6ad9f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0d4973305911494fa1000faf75c6ad9f 2024-11-24T11:15:31,444 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0d4973305911494fa1000faf75c6ad9f, entries=150, sequenceid=131, filesize=11.7 K 2024-11-24T11:15:31,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/2e49fb6336f748c9b729e4b48e652133 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2e49fb6336f748c9b729e4b48e652133 2024-11-24T11:15:31,449 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2e49fb6336f748c9b729e4b48e652133, entries=150, sequenceid=131, filesize=11.7 K 2024-11-24T11:15:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/c139d74ddc5244898a436e8442900108 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/c139d74ddc5244898a436e8442900108 2024-11-24T11:15:31,453 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/c139d74ddc5244898a436e8442900108, entries=150, sequenceid=131, filesize=11.7 K 2024-11-24T11:15:31,454 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76c783a6d487b45ef515b468f5ea8d41 in 507ms, sequenceid=131, compaction requested=true 2024-11-24T11:15:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-24T11:15:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-24T11:15:31,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T11:15:31,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2590 sec 2024-11-24T11:15:31,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.2630 sec 2024-11-24T11:15:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:31,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=134.18 KB heapSize=383.70 KB 2024-11-24T11:15:31,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:31,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:31,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:31,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:31,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:31,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:31,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/225450ae466248cbb3e0081575b671ee is 50, key is test_row_0/A:col10/1732446931488/Put/seqid=0 2024-11-24T11:15:31,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742098_1274 (size=16931) 2024-11-24T11:15:31,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/225450ae466248cbb3e0081575b671ee 2024-11-24T11:15:31,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/38709b91ba0e47898824f4019401c4ea is 50, key is test_row_0/B:col10/1732446931488/Put/seqid=0 2024-11-24T11:15:31,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446991505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742099_1275 (size=12151) 2024-11-24T11:15:31,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446991515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446991516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446991624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446991634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446991634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446991763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,769 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446991774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,779 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446991833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446991844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446991844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:31,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/38709b91ba0e47898824f4019401c4ea 2024-11-24T11:15:31,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/435d0b27aa8743ab8a6493cbd1037452 is 50, key is test_row_0/C:col10/1732446931488/Put/seqid=0 2024-11-24T11:15:31,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742100_1276 (size=12151) 2024-11-24T11:15:31,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/435d0b27aa8743ab8a6493cbd1037452 2024-11-24T11:15:31,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/225450ae466248cbb3e0081575b671ee as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/225450ae466248cbb3e0081575b671ee 2024-11-24T11:15:31,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/225450ae466248cbb3e0081575b671ee, entries=250, sequenceid=163, filesize=16.5 K 2024-11-24T11:15:31,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/38709b91ba0e47898824f4019401c4ea as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/38709b91ba0e47898824f4019401c4ea 2024-11-24T11:15:31,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/38709b91ba0e47898824f4019401c4ea, entries=150, sequenceid=163, filesize=11.9 K 2024-11-24T11:15:31,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/435d0b27aa8743ab8a6493cbd1037452 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/435d0b27aa8743ab8a6493cbd1037452 2024-11-24T11:15:31,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/435d0b27aa8743ab8a6493cbd1037452, entries=150, sequenceid=163, filesize=11.9 K 2024-11-24T11:15:31,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~403.58 KB/413264, currentSize=40.25 KB/41220 for 76c783a6d487b45ef515b468f5ea8d41 in 496ms, sequenceid=163, compaction requested=true 2024-11-24T11:15:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:31,986 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:31,987 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:31,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:31,988 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:31,988 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/A is initiating minor compaction (all files) 2024-11-24T11:15:31,988 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/A in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:31,988 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:31,988 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/B is initiating minor compaction (all files) 2024-11-24T11:15:31,988 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d5acb9bf927444d7afa1c7812e9889e2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a755ee63e78341b2a292bdd292bb15e8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0d4973305911494fa1000faf75c6ad9f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/225450ae466248cbb3e0081575b671ee] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=54.2 K 2024-11-24T11:15:31,988 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/B in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:31,988 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/d20af67afff94dfbaea9534ebb952a28, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/da3ab028553641bfbf81579e9589282d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2e49fb6336f748c9b729e4b48e652133, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/38709b91ba0e47898824f4019401c4ea] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=47.2 K 2024-11-24T11:15:31,989 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d5acb9bf927444d7afa1c7812e9889e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446928280 2024-11-24T11:15:31,989 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20af67afff94dfbaea9534ebb952a28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446928280 2024-11-24T11:15:31,989 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a755ee63e78341b2a292bdd292bb15e8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732446929596 2024-11-24T11:15:31,989 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting da3ab028553641bfbf81579e9589282d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732446929596 2024-11-24T11:15:31,989 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d4973305911494fa1000faf75c6ad9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732446930449 2024-11-24T11:15:31,989 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e49fb6336f748c9b729e4b48e652133, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732446930449 2024-11-24T11:15:31,990 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 225450ae466248cbb3e0081575b671ee, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732446931157 2024-11-24T11:15:31,990 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38709b91ba0e47898824f4019401c4ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732446931158 2024-11-24T11:15:32,017 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:32,018 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/2fc9ac5bbd6b4ef08076c84030dff649 is 50, key is test_row_0/A:col10/1732446931488/Put/seqid=0 2024-11-24T11:15:32,039 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:32,040 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f721666d663449eabbed63342de13017 is 50, key is test_row_0/B:col10/1732446931488/Put/seqid=0 2024-11-24T11:15:32,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742101_1277 (size=12493) 2024-11-24T11:15:32,055 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/2fc9ac5bbd6b4ef08076c84030dff649 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/2fc9ac5bbd6b4ef08076c84030dff649 2024-11-24T11:15:32,059 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/A of 76c783a6d487b45ef515b468f5ea8d41 into 2fc9ac5bbd6b4ef08076c84030dff649(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:32,059 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:32,059 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/A, priority=12, startTime=1732446931986; duration=0sec 2024-11-24T11:15:32,060 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:32,060 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:A 2024-11-24T11:15:32,060 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:32,061 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:32,061 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/C is initiating minor compaction (all files) 2024-11-24T11:15:32,061 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/C in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,061 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8f7dfc7c3ea146a8a3a13c526ce1f39e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/49243d09315542d281b510fc08f5599d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/c139d74ddc5244898a436e8442900108, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/435d0b27aa8743ab8a6493cbd1037452] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=47.2 K 2024-11-24T11:15:32,062 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7dfc7c3ea146a8a3a13c526ce1f39e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732446928280 2024-11-24T11:15:32,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742102_1278 (size=12493) 2024-11-24T11:15:32,063 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 49243d09315542d281b510fc08f5599d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732446929596 2024-11-24T11:15:32,063 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c139d74ddc5244898a436e8442900108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732446930449 2024-11-24T11:15:32,064 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 435d0b27aa8743ab8a6493cbd1037452, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732446931158 2024-11-24T11:15:32,071 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:32,072 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/629fac6f05aa4ad6a2f72a7a7ccbeeb1 is 50, key is test_row_0/C:col10/1732446931488/Put/seqid=0 2024-11-24T11:15:32,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742103_1279 (size=12493) 2024-11-24T11:15:32,081 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/629fac6f05aa4ad6a2f72a7a7ccbeeb1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/629fac6f05aa4ad6a2f72a7a7ccbeeb1 2024-11-24T11:15:32,085 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/C of 76c783a6d487b45ef515b468f5ea8d41 into 629fac6f05aa4ad6a2f72a7a7ccbeeb1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:32,085 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:32,085 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/C, priority=12, startTime=1732446931986; duration=0sec 2024-11-24T11:15:32,085 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:32,085 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:C 2024-11-24T11:15:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:32,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:32,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/9c95b7496367485a927840f8fa22fb54 is 50, key is test_row_0/A:col10/1732446931500/Put/seqid=0 2024-11-24T11:15:32,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742104_1280 (size=9757) 2024-11-24T11:15:32,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446992173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446932170, max timestamp=1732446932175Num uniques 859; 2024-11-24T11:15:32,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:32,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446932170, max timestamp=1732446932175Num uniques 859; 2024-11-24T11:15:32,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:32,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446932170, max timestamp=1732446932175Num uniques 859; 2024-11-24T11:15:32,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:32,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446992219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446992220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446992284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446992333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446992333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T11:15:32,364 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:32,364 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-24T11:15:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T11:15:32,366 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:32,367 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:32,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:32,467 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f721666d663449eabbed63342de13017 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f721666d663449eabbed63342de13017 2024-11-24T11:15:32,472 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/B of 76c783a6d487b45ef515b468f5ea8d41 into f721666d663449eabbed63342de13017(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:32,472 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:32,472 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/B, priority=12, startTime=1732446931986; duration=0sec 2024-11-24T11:15:32,472 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:32,472 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:B 2024-11-24T11:15:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T11:15:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446992493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T11:15:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446992543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446992543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/9c95b7496367485a927840f8fa22fb54 2024-11-24T11:15:32,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/a71902dedd7e4fe49a55fec90d58bd65 is 50, key is test_row_0/B:col10/1732446931500/Put/seqid=0 2024-11-24T11:15:32,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742105_1281 (size=9757) 2024-11-24T11:15:32,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T11:15:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T11:15:32,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446992804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T11:15:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446992853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446992853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:32,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T11:15:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:32,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:32,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/a71902dedd7e4fe49a55fec90d58bd65 2024-11-24T11:15:32,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/a044eb8dff444729b36b9f54dad3ea0d is 50, key is test_row_0/C:col10/1732446931500/Put/seqid=0 2024-11-24T11:15:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742106_1282 (size=9757) 2024-11-24T11:15:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T11:15:33,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T11:15:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:33,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:33,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T11:15:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:33,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:33,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446993314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:33,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446993363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:33,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:33,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446993363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:33,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:33,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:33,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/a044eb8dff444729b36b9f54dad3ea0d 2024-11-24T11:15:33,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/9c95b7496367485a927840f8fa22fb54 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/9c95b7496367485a927840f8fa22fb54 2024-11-24T11:15:33,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/9c95b7496367485a927840f8fa22fb54, entries=100, sequenceid=175, filesize=9.5 K 2024-11-24T11:15:33,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/a71902dedd7e4fe49a55fec90d58bd65 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/a71902dedd7e4fe49a55fec90d58bd65 2024-11-24T11:15:33,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/a71902dedd7e4fe49a55fec90d58bd65, entries=100, sequenceid=175, filesize=9.5 K 2024-11-24T11:15:33,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/a044eb8dff444729b36b9f54dad3ea0d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a044eb8dff444729b36b9f54dad3ea0d 2024-11-24T11:15:33,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a044eb8dff444729b36b9f54dad3ea0d, entries=100, sequenceid=175, filesize=9.5 K 2024-11-24T11:15:33,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 76c783a6d487b45ef515b468f5ea8d41 in 1258ms, sequenceid=175, compaction requested=false 2024-11-24T11:15:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:33,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T11:15:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:33,434 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T11:15:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/6745d3a78b6b4b72a671adbfc209ef58 is 50, key is test_row_0/A:col10/1732446932217/Put/seqid=0 2024-11-24T11:15:33,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742107_1283 (size=12151) 2024-11-24T11:15:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T11:15:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:33,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446993809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:33,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446993812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:33,843 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/6745d3a78b6b4b72a671adbfc209ef58 2024-11-24T11:15:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/5c449523823e44709be16300c099482f is 50, key is test_row_0/B:col10/1732446932217/Put/seqid=0 2024-11-24T11:15:33,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742108_1284 (size=12151) 2024-11-24T11:15:33,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446993924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:33,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446993924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446994133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:34,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446994134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,255 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/5c449523823e44709be16300c099482f 2024-11-24T11:15:34,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e30d48d6cf1c48f2b4f2b437de9b2720 is 50, key is test_row_0/C:col10/1732446932217/Put/seqid=0 2024-11-24T11:15:34,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742109_1285 (size=12151) 2024-11-24T11:15:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446994333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:34,337 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446994374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:34,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446994374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:34,378 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:34,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,378 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:34,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446994443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446994444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:34,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:34,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T11:15:34,667 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e30d48d6cf1c48f2b4f2b437de9b2720 2024-11-24T11:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/6745d3a78b6b4b72a671adbfc209ef58 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6745d3a78b6b4b72a671adbfc209ef58 2024-11-24T11:15:34,674 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6745d3a78b6b4b72a671adbfc209ef58, entries=150, sequenceid=213, filesize=11.9 K 2024-11-24T11:15:34,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/5c449523823e44709be16300c099482f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/5c449523823e44709be16300c099482f 2024-11-24T11:15:34,678 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/5c449523823e44709be16300c099482f, entries=150, sequenceid=213, filesize=11.9 K 2024-11-24T11:15:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/e30d48d6cf1c48f2b4f2b437de9b2720 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e30d48d6cf1c48f2b4f2b437de9b2720 2024-11-24T11:15:34,683 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e30d48d6cf1c48f2b4f2b437de9b2720, entries=150, sequenceid=213, filesize=11.9 K 2024-11-24T11:15:34,683 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 76c783a6d487b45ef515b468f5ea8d41 in 1249ms, sequenceid=213, compaction requested=true 2024-11-24T11:15:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:34,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:34,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-24T11:15:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-24T11:15:34,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-24T11:15:34,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3180 sec 2024-11-24T11:15:34,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.3220 sec 2024-11-24T11:15:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:34,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T11:15:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:34,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/8ea111dd047e4ac5b06ee392c4a76cfd is 50, key is test_row_0/A:col10/1732446934956/Put/seqid=0 2024-11-24T11:15:34,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742110_1286 (size=14541) 2024-11-24T11:15:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446995014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446995015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446995124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446995124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446995333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446995333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/8ea111dd047e4ac5b06ee392c4a76cfd 2024-11-24T11:15:35,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/726bd2b22e964631a48b5ba84781b994 is 50, key is test_row_0/B:col10/1732446934956/Put/seqid=0 2024-11-24T11:15:35,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742111_1287 (size=12151) 2024-11-24T11:15:35,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446995644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446995644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:35,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/726bd2b22e964631a48b5ba84781b994 2024-11-24T11:15:35,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/2ac5009118aa49e59b634b88fe3aeeaa is 50, key is test_row_0/C:col10/1732446934956/Put/seqid=0 2024-11-24T11:15:35,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742112_1288 (size=12151) 2024-11-24T11:15:35,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/2ac5009118aa49e59b634b88fe3aeeaa 2024-11-24T11:15:35,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/8ea111dd047e4ac5b06ee392c4a76cfd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/8ea111dd047e4ac5b06ee392c4a76cfd 2024-11-24T11:15:35,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/8ea111dd047e4ac5b06ee392c4a76cfd, entries=200, sequenceid=225, filesize=14.2 K 2024-11-24T11:15:35,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/726bd2b22e964631a48b5ba84781b994 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/726bd2b22e964631a48b5ba84781b994 2024-11-24T11:15:35,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/726bd2b22e964631a48b5ba84781b994, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T11:15:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/2ac5009118aa49e59b634b88fe3aeeaa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/2ac5009118aa49e59b634b88fe3aeeaa 2024-11-24T11:15:35,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/2ac5009118aa49e59b634b88fe3aeeaa, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T11:15:35,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 76c783a6d487b45ef515b468f5ea8d41 in 855ms, sequenceid=225, compaction requested=true 2024-11-24T11:15:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:35,811 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:35,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T11:15:35,812 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:35,813 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48942 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:35,813 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/A is initiating minor compaction (all files) 2024-11-24T11:15:35,813 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/A in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:35,813 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/2fc9ac5bbd6b4ef08076c84030dff649, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/9c95b7496367485a927840f8fa22fb54, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6745d3a78b6b4b72a671adbfc209ef58, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/8ea111dd047e4ac5b06ee392c4a76cfd] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=47.8 K 2024-11-24T11:15:35,813 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc9ac5bbd6b4ef08076c84030dff649, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732446931158 2024-11-24T11:15:35,814 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:35,814 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/B is initiating minor compaction (all files) 2024-11-24T11:15:35,814 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/B in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:35,814 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f721666d663449eabbed63342de13017, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/a71902dedd7e4fe49a55fec90d58bd65, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/5c449523823e44709be16300c099482f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/726bd2b22e964631a48b5ba84781b994] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=45.5 K 2024-11-24T11:15:35,815 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c95b7496367485a927840f8fa22fb54, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732446931500 2024-11-24T11:15:35,815 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f721666d663449eabbed63342de13017, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732446931158 2024-11-24T11:15:35,815 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6745d3a78b6b4b72a671adbfc209ef58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732446932209 2024-11-24T11:15:35,815 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a71902dedd7e4fe49a55fec90d58bd65, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732446931500 2024-11-24T11:15:35,815 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ea111dd047e4ac5b06ee392c4a76cfd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732446933797 2024-11-24T11:15:35,815 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c449523823e44709be16300c099482f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732446932209 2024-11-24T11:15:35,816 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 726bd2b22e964631a48b5ba84781b994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732446933797 2024-11-24T11:15:35,824 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:35,824 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/4340e746cf6e4cefb3befd6598e897c9 is 50, key is test_row_0/B:col10/1732446934956/Put/seqid=0 2024-11-24T11:15:35,827 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:35,828 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/a86c39040b534b49aa558295658b5c7d is 50, key is test_row_0/A:col10/1732446934956/Put/seqid=0 2024-11-24T11:15:35,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742113_1289 (size=12629) 2024-11-24T11:15:35,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742114_1290 (size=12629) 2024-11-24T11:15:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:36,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T11:15:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:36,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/e25191d8a0a9471c9ab6c52491930f00 is 50, key is test_row_0/A:col10/1732446936156/Put/seqid=0 2024-11-24T11:15:36,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742115_1291 (size=12151) 2024-11-24T11:15:36,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446996173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446996178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,237 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/4340e746cf6e4cefb3befd6598e897c9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/4340e746cf6e4cefb3befd6598e897c9 2024-11-24T11:15:36,238 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/a86c39040b534b49aa558295658b5c7d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a86c39040b534b49aa558295658b5c7d 2024-11-24T11:15:36,242 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/B of 76c783a6d487b45ef515b468f5ea8d41 into 4340e746cf6e4cefb3befd6598e897c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:36,242 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:36,242 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/B, priority=12, startTime=1732446935811; duration=0sec 2024-11-24T11:15:36,242 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/A of 76c783a6d487b45ef515b468f5ea8d41 into a86c39040b534b49aa558295658b5c7d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:36,242 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:36,242 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/A, priority=12, startTime=1732446935811; duration=0sec 2024-11-24T11:15:36,242 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:36,242 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:36,242 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:B 2024-11-24T11:15:36,242 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:A 2024-11-24T11:15:36,242 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:36,243 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:36,243 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/C is initiating minor compaction (all files) 2024-11-24T11:15:36,243 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/C in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:36,244 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/629fac6f05aa4ad6a2f72a7a7ccbeeb1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a044eb8dff444729b36b9f54dad3ea0d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e30d48d6cf1c48f2b4f2b437de9b2720, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/2ac5009118aa49e59b634b88fe3aeeaa] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=45.5 K 2024-11-24T11:15:36,244 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 629fac6f05aa4ad6a2f72a7a7ccbeeb1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732446931158 2024-11-24T11:15:36,244 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a044eb8dff444729b36b9f54dad3ea0d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732446931500 2024-11-24T11:15:36,244 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e30d48d6cf1c48f2b4f2b437de9b2720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732446932209 2024-11-24T11:15:36,245 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac5009118aa49e59b634b88fe3aeeaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732446933797 2024-11-24T11:15:36,251 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:36,252 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/709649a836ee4c0e85c5c83fe83c5629 is 50, key is test_row_0/C:col10/1732446934956/Put/seqid=0 2024-11-24T11:15:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742116_1292 (size=12629) 2024-11-24T11:15:36,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446996283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446996294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732446996353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,356 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732446996383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732446996383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,386 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,386 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446996493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446996503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T11:15:36,533 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:36,534 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-24T11:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T11:15:36,535 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:36,536 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:36,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:36,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/e25191d8a0a9471c9ab6c52491930f00 2024-11-24T11:15:36,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/3a3169c2dd9c4866adcac78dc044d8b8 is 50, key is test_row_0/B:col10/1732446936156/Put/seqid=0 2024-11-24T11:15:36,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742117_1293 (size=12151) 2024-11-24T11:15:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T11:15:36,660 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/709649a836ee4c0e85c5c83fe83c5629 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/709649a836ee4c0e85c5c83fe83c5629 2024-11-24T11:15:36,663 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/C of 76c783a6d487b45ef515b468f5ea8d41 into 709649a836ee4c0e85c5c83fe83c5629(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:36,663 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:36,663 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/C, priority=12, startTime=1732446935811; duration=0sec 2024-11-24T11:15:36,664 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:36,664 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:C 2024-11-24T11:15:36,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T11:15:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:36,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446996803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446996813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:36,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T11:15:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:36,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T11:15:36,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/3a3169c2dd9c4866adcac78dc044d8b8 2024-11-24T11:15:36,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/87e76f08f6c240a1a09772974291e940 is 50, key is test_row_0/C:col10/1732446936156/Put/seqid=0 2024-11-24T11:15:36,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742118_1294 (size=12151) 2024-11-24T11:15:36,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T11:15:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:36,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:37,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T11:15:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:37,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T11:15:37,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T11:15:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:37,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:37,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446997313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446997323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:37,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/87e76f08f6c240a1a09772974291e940 2024-11-24T11:15:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/e25191d8a0a9471c9ab6c52491930f00 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e25191d8a0a9471c9ab6c52491930f00 2024-11-24T11:15:37,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e25191d8a0a9471c9ab6c52491930f00, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T11:15:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/3a3169c2dd9c4866adcac78dc044d8b8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3a3169c2dd9c4866adcac78dc044d8b8 2024-11-24T11:15:37,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3a3169c2dd9c4866adcac78dc044d8b8, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T11:15:37,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/87e76f08f6c240a1a09772974291e940 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/87e76f08f6c240a1a09772974291e940 2024-11-24T11:15:37,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/87e76f08f6c240a1a09772974291e940, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T11:15:37,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 76c783a6d487b45ef515b468f5ea8d41 in 1246ms, sequenceid=247, compaction requested=false 2024-11-24T11:15:37,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:37,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T11:15:37,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:37,452 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:37,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:37,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/be4f37b0b5e442c7bc0a2ec43786960c is 50, key is test_row_0/A:col10/1732446936177/Put/seqid=0 2024-11-24T11:15:37,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742119_1295 (size=12151) 2024-11-24T11:15:37,460 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/be4f37b0b5e442c7bc0a2ec43786960c 2024-11-24T11:15:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/3756c282df7747cca90de4b3e2d240ab is 50, key is test_row_0/B:col10/1732446936177/Put/seqid=0 2024-11-24T11:15:37,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742120_1296 (size=12151) 2024-11-24T11:15:37,474 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/3756c282df7747cca90de4b3e2d240ab 2024-11-24T11:15:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/b669e69a038f49c68e24d2e36d46f403 is 50, key is test_row_0/C:col10/1732446936177/Put/seqid=0 2024-11-24T11:15:37,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742121_1297 (size=12151) 2024-11-24T11:15:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T11:15:37,889 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/b669e69a038f49c68e24d2e36d46f403 2024-11-24T11:15:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/be4f37b0b5e442c7bc0a2ec43786960c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/be4f37b0b5e442c7bc0a2ec43786960c 2024-11-24T11:15:37,897 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/be4f37b0b5e442c7bc0a2ec43786960c, entries=150, sequenceid=260, filesize=11.9 K 2024-11-24T11:15:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/3756c282df7747cca90de4b3e2d240ab as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3756c282df7747cca90de4b3e2d240ab 2024-11-24T11:15:37,902 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3756c282df7747cca90de4b3e2d240ab, entries=150, sequenceid=260, filesize=11.9 K 2024-11-24T11:15:37,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/b669e69a038f49c68e24d2e36d46f403 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b669e69a038f49c68e24d2e36d46f403 2024-11-24T11:15:37,906 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b669e69a038f49c68e24d2e36d46f403, entries=150, sequenceid=260, filesize=11.9 K 2024-11-24T11:15:37,907 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 76c783a6d487b45ef515b468f5ea8d41 in 455ms, sequenceid=260, compaction requested=true 2024-11-24T11:15:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-24T11:15:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-24T11:15:37,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-24T11:15:37,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3730 sec 2024-11-24T11:15:37,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.3770 sec 2024-11-24T11:15:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:38,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:38,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/5dfd46149db24704bee7a7654abdc22c is 50, key is test_row_0/A:col10/1732446938330/Put/seqid=0 2024-11-24T11:15:38,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742122_1298 (size=14741) 2024-11-24T11:15:38,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446998380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446938367, max timestamp=1732446938382Num uniques 859; 2024-11-24T11:15:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:38,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:38,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446938367, max timestamp=1732446938382Num uniques 859; 2024-11-24T11:15:38,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:38,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732446938367, max timestamp=1732446938382Num uniques 859; 2024-11-24T11:15:38,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:15:38,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446998459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:38,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446998493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446998574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T11:15:38,684 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:38,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-24T11:15:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T11:15:38,686 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:38,686 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:38,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:38,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446998704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:38,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:38,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/5dfd46149db24704bee7a7654abdc22c 2024-11-24T11:15:38,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/155f1fb955f14e7eb2768ba3ae4eb758 is 50, key is test_row_0/B:col10/1732446938330/Put/seqid=0 2024-11-24T11:15:38,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742123_1299 (size=12301) 2024-11-24T11:15:38,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446998784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T11:15:38,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T11:15:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:38,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:38,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T11:15:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:38,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T11:15:39,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446999014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:39,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:39,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446999093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:39,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T11:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/155f1fb955f14e7eb2768ba3ae4eb758 2024-11-24T11:15:39,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/79be35b36f1f4301978c0ae0ac56a821 is 50, key is test_row_0/C:col10/1732446938330/Put/seqid=0 2024-11-24T11:15:39,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742124_1300 (size=12301) 2024-11-24T11:15:39,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T11:15:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T11:15:39,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T11:15:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732446999524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:39,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/79be35b36f1f4301978c0ae0ac56a821 2024-11-24T11:15:39,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/5dfd46149db24704bee7a7654abdc22c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/5dfd46149db24704bee7a7654abdc22c 2024-11-24T11:15:39,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/5dfd46149db24704bee7a7654abdc22c, entries=200, sequenceid=271, filesize=14.4 K 2024-11-24T11:15:39,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/155f1fb955f14e7eb2768ba3ae4eb758 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/155f1fb955f14e7eb2768ba3ae4eb758 2024-11-24T11:15:39,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/155f1fb955f14e7eb2768ba3ae4eb758, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T11:15:39,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/79be35b36f1f4301978c0ae0ac56a821 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/79be35b36f1f4301978c0ae0ac56a821 2024-11-24T11:15:39,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/79be35b36f1f4301978c0ae0ac56a821, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T11:15:39,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76c783a6d487b45ef515b468f5ea8d41 in 1244ms, sequenceid=271, compaction requested=true 2024-11-24T11:15:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:39,583 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:39,583 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:39,584 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:39,584 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51672 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:39,584 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/A is initiating minor compaction (all files) 2024-11-24T11:15:39,584 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/B is initiating minor compaction (all files) 2024-11-24T11:15:39,584 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/B in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,584 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/A in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,584 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/4340e746cf6e4cefb3befd6598e897c9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3a3169c2dd9c4866adcac78dc044d8b8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3756c282df7747cca90de4b3e2d240ab, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/155f1fb955f14e7eb2768ba3ae4eb758] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=48.1 K 2024-11-24T11:15:39,584 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a86c39040b534b49aa558295658b5c7d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e25191d8a0a9471c9ab6c52491930f00, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/be4f37b0b5e442c7bc0a2ec43786960c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/5dfd46149db24704bee7a7654abdc22c] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=50.5 K 2024-11-24T11:15:39,585 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4340e746cf6e4cefb3befd6598e897c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732446933797 2024-11-24T11:15:39,585 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting a86c39040b534b49aa558295658b5c7d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732446933797 2024-11-24T11:15:39,585 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25191d8a0a9471c9ab6c52491930f00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732446935008 2024-11-24T11:15:39,585 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a3169c2dd9c4866adcac78dc044d8b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732446935008 2024-11-24T11:15:39,585 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3756c282df7747cca90de4b3e2d240ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732446936167 2024-11-24T11:15:39,585 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting be4f37b0b5e442c7bc0a2ec43786960c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732446936167 2024-11-24T11:15:39,586 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 155f1fb955f14e7eb2768ba3ae4eb758, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732446938330 2024-11-24T11:15:39,586 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dfd46149db24704bee7a7654abdc22c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732446938330 2024-11-24T11:15:39,593 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:39,594 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/25e2441aebb44be6a44db130c997d83a is 50, key is test_row_0/B:col10/1732446938330/Put/seqid=0 2024-11-24T11:15:39,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742125_1301 (size=12915) 2024-11-24T11:15:39,597 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:39,598 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/7fb2275fccc44efb9f48bda79696072c is 50, key is test_row_0/A:col10/1732446938330/Put/seqid=0 2024-11-24T11:15:39,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742126_1302 (size=12915) 2024-11-24T11:15:39,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T11:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,603 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T11:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:39,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:39,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:39,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d02bf03e11d742ef90913cdc726f0c38 is 50, key is test_row_0/A:col10/1732446938451/Put/seqid=0 2024-11-24T11:15:39,610 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/7fb2275fccc44efb9f48bda79696072c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/7fb2275fccc44efb9f48bda79696072c 2024-11-24T11:15:39,620 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/A of 76c783a6d487b45ef515b468f5ea8d41 into 7fb2275fccc44efb9f48bda79696072c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:39,620 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:39,620 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/A, priority=12, startTime=1732446939583; duration=0sec 2024-11-24T11:15:39,620 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:39,620 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:A 2024-11-24T11:15:39,620 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:15:39,622 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:15:39,622 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/C is initiating minor compaction (all files) 2024-11-24T11:15:39,622 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/C in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,623 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/709649a836ee4c0e85c5c83fe83c5629, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/87e76f08f6c240a1a09772974291e940, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b669e69a038f49c68e24d2e36d46f403, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/79be35b36f1f4301978c0ae0ac56a821] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=48.1 K 2024-11-24T11:15:39,623 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709649a836ee4c0e85c5c83fe83c5629, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732446933797 2024-11-24T11:15:39,623 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e76f08f6c240a1a09772974291e940, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732446935008 2024-11-24T11:15:39,624 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b669e69a038f49c68e24d2e36d46f403, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732446936167 2024-11-24T11:15:39,625 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79be35b36f1f4301978c0ae0ac56a821, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732446938330 2024-11-24T11:15:39,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742127_1303 (size=12301) 2024-11-24T11:15:39,638 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d02bf03e11d742ef90913cdc726f0c38 2024-11-24T11:15:39,640 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#C#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:39,640 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/278fd0ea3bf74281a5f6a7aaed09a2aa is 50, key is test_row_0/C:col10/1732446938330/Put/seqid=0 2024-11-24T11:15:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/681ea80dda56481ebc50fa76a699a9e8 is 50, key is test_row_0/B:col10/1732446938451/Put/seqid=0 2024-11-24T11:15:39,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742128_1304 (size=12915) 2024-11-24T11:15:39,657 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/278fd0ea3bf74281a5f6a7aaed09a2aa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/278fd0ea3bf74281a5f6a7aaed09a2aa 2024-11-24T11:15:39,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446999654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:39,662 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/C of 76c783a6d487b45ef515b468f5ea8d41 into 278fd0ea3bf74281a5f6a7aaed09a2aa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:39,662 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:39,662 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/C, priority=12, startTime=1732446939583; duration=0sec 2024-11-24T11:15:39,662 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:39,662 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:C 2024-11-24T11:15:39,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742129_1305 (size=12301) 2024-11-24T11:15:39,673 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/681ea80dda56481ebc50fa76a699a9e8 2024-11-24T11:15:39,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/b84e28774b224d96b4fb74e6f369af5d is 50, key is test_row_0/C:col10/1732446938451/Put/seqid=0 2024-11-24T11:15:39,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742130_1306 (size=12301) 2024-11-24T11:15:39,687 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/b84e28774b224d96b4fb74e6f369af5d 2024-11-24T11:15:39,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/d02bf03e11d742ef90913cdc726f0c38 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d02bf03e11d742ef90913cdc726f0c38 2024-11-24T11:15:39,693 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d02bf03e11d742ef90913cdc726f0c38, entries=150, sequenceid=306, filesize=12.0 K 2024-11-24T11:15:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/681ea80dda56481ebc50fa76a699a9e8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/681ea80dda56481ebc50fa76a699a9e8 2024-11-24T11:15:39,697 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/681ea80dda56481ebc50fa76a699a9e8, entries=150, sequenceid=306, filesize=12.0 K 2024-11-24T11:15:39,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/b84e28774b224d96b4fb74e6f369af5d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b84e28774b224d96b4fb74e6f369af5d 2024-11-24T11:15:39,701 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b84e28774b224d96b4fb74e6f369af5d, entries=150, sequenceid=306, filesize=12.0 K 2024-11-24T11:15:39,701 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 76c783a6d487b45ef515b468f5ea8d41 in 98ms, sequenceid=306, compaction requested=false 2024-11-24T11:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-24T11:15:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-24T11:15:39,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T11:15:39,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-24T11:15:39,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0210 sec 2024-11-24T11:15:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:39,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:15:39,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:39,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:39,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:39,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:39,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:39,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:39,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/e6f6764a32e4424ba9acfd0322f7c937 is 50, key is test_row_0/A:col10/1732446939623/Put/seqid=0 2024-11-24T11:15:39,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742131_1307 (size=14741) 2024-11-24T11:15:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T11:15:39,824 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:39,824 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-24T11:15:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T11:15:39,825 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:39,826 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:39,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:39,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446999843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T11:15:39,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732446999953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:39,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:39,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:39,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:39,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,001 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/25e2441aebb44be6a44db130c997d83a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/25e2441aebb44be6a44db130c997d83a 2024-11-24T11:15:40,006 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/B of 76c783a6d487b45ef515b468f5ea8d41 into 25e2441aebb44be6a44db130c997d83a(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:40,006 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:40,006 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/B, priority=12, startTime=1732446939583; duration=0sec 2024-11-24T11:15:40,006 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:40,006 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:B 2024-11-24T11:15:40,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:40,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T11:15:40,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447000163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:40,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/e6f6764a32e4424ba9acfd0322f7c937 2024-11-24T11:15:40,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/bcfab545bd5c42a3900fde1a6654bc3f is 50, key is test_row_0/B:col10/1732446939623/Put/seqid=0 2024-11-24T11:15:40,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742132_1308 (size=12301) 2024-11-24T11:15:40,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:40,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:40,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37272 deadline: 1732447000384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:40,385 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37332 deadline: 1732447000394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:40,397 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37304 deadline: 1732447000413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:40,417 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:40,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:40,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:40,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T11:15:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447000474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:40,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732447000533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:40,536 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:40,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/bcfab545bd5c42a3900fde1a6654bc3f 2024-11-24T11:15:40,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:40,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:40,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/95474c6d952b4e47b1df401e3d9a9648 is 50, key is test_row_0/C:col10/1732446939623/Put/seqid=0 2024-11-24T11:15:40,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742133_1309 (size=12301) 2024-11-24T11:15:40,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:40,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:40,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:40,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:40,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:40,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T11:15:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447000983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:40,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/95474c6d952b4e47b1df401e3d9a9648 2024-11-24T11:15:40,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/e6f6764a32e4424ba9acfd0322f7c937 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e6f6764a32e4424ba9acfd0322f7c937 2024-11-24T11:15:41,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e6f6764a32e4424ba9acfd0322f7c937, entries=200, sequenceid=320, filesize=14.4 K 2024-11-24T11:15:41,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/bcfab545bd5c42a3900fde1a6654bc3f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/bcfab545bd5c42a3900fde1a6654bc3f 2024-11-24T11:15:41,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/bcfab545bd5c42a3900fde1a6654bc3f, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T11:15:41,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/95474c6d952b4e47b1df401e3d9a9648 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/95474c6d952b4e47b1df401e3d9a9648 2024-11-24T11:15:41,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/95474c6d952b4e47b1df401e3d9a9648, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T11:15:41,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 76c783a6d487b45ef515b468f5ea8d41 in 1243ms, sequenceid=320, compaction requested=true 2024-11-24T11:15:41,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:41,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:41,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:41,010 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:41,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:41,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:41,011 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:41,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:41,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:41,011 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:41,011 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:41,011 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/B is initiating minor compaction (all files) 2024-11-24T11:15:41,011 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/A is initiating minor compaction (all files) 2024-11-24T11:15:41,011 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/B in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:41,011 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/A in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:41,011 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/25e2441aebb44be6a44db130c997d83a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/681ea80dda56481ebc50fa76a699a9e8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/bcfab545bd5c42a3900fde1a6654bc3f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=36.6 K 2024-11-24T11:15:41,011 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/7fb2275fccc44efb9f48bda79696072c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d02bf03e11d742ef90913cdc726f0c38, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e6f6764a32e4424ba9acfd0322f7c937] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=39.0 K 2024-11-24T11:15:41,012 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 25e2441aebb44be6a44db130c997d83a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732446938330 2024-11-24T11:15:41,012 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fb2275fccc44efb9f48bda79696072c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732446938330 2024-11-24T11:15:41,012 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 681ea80dda56481ebc50fa76a699a9e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732446938420 2024-11-24T11:15:41,012 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02bf03e11d742ef90913cdc726f0c38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732446938420 2024-11-24T11:15:41,012 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f6764a32e4424ba9acfd0322f7c937, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732446939623 2024-11-24T11:15:41,012 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting bcfab545bd5c42a3900fde1a6654bc3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732446939623 2024-11-24T11:15:41,019 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:41,019 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/ad095b879d6f4388a1c375129abf0cdb is 50, key is test_row_0/B:col10/1732446939623/Put/seqid=0 2024-11-24T11:15:41,019 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:41,020 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/4446c5f198f04f3ab0d6fc99bac2136d is 50, key is test_row_0/A:col10/1732446939623/Put/seqid=0 2024-11-24T11:15:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742134_1310 (size=13017) 2024-11-24T11:15:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742135_1311 (size=13017) 2024-11-24T11:15:41,034 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/ad095b879d6f4388a1c375129abf0cdb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ad095b879d6f4388a1c375129abf0cdb 2024-11-24T11:15:41,039 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/B of 76c783a6d487b45ef515b468f5ea8d41 into ad095b879d6f4388a1c375129abf0cdb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:41,039 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:41,039 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/B, priority=13, startTime=1732446941010; duration=0sec 2024-11-24T11:15:41,039 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:41,039 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:B 2024-11-24T11:15:41,040 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:41,041 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:41,041 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/C is initiating minor compaction (all files) 2024-11-24T11:15:41,041 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/C in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:41,041 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/278fd0ea3bf74281a5f6a7aaed09a2aa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b84e28774b224d96b4fb74e6f369af5d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/95474c6d952b4e47b1df401e3d9a9648] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=36.6 K 2024-11-24T11:15:41,042 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 278fd0ea3bf74281a5f6a7aaed09a2aa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732446938330 2024-11-24T11:15:41,042 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b84e28774b224d96b4fb74e6f369af5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732446938420 2024-11-24T11:15:41,042 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 95474c6d952b4e47b1df401e3d9a9648, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732446939623 2024-11-24T11:15:41,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T11:15:41,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:41,048 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T11:15:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:41,049 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:41,049 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/308f0e80ca474a45b35b38a9dc18d625 is 50, key is test_row_0/C:col10/1732446939623/Put/seqid=0 2024-11-24T11:15:41,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/c1d47842d7054c1195e8f8dc47f1ed7a is 50, key is test_row_0/A:col10/1732446939842/Put/seqid=0 2024-11-24T11:15:41,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742136_1312 (size=13017) 2024-11-24T11:15:41,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742137_1313 (size=12301) 2024-11-24T11:15:41,490 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/c1d47842d7054c1195e8f8dc47f1ed7a 2024-11-24T11:15:41,502 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/308f0e80ca474a45b35b38a9dc18d625 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/308f0e80ca474a45b35b38a9dc18d625 2024-11-24T11:15:41,503 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/4446c5f198f04f3ab0d6fc99bac2136d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/4446c5f198f04f3ab0d6fc99bac2136d 2024-11-24T11:15:41,508 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/C of 76c783a6d487b45ef515b468f5ea8d41 into 308f0e80ca474a45b35b38a9dc18d625(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:41,508 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/A of 76c783a6d487b45ef515b468f5ea8d41 into 4446c5f198f04f3ab0d6fc99bac2136d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:41,508 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:41,508 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:41,508 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/C, priority=13, startTime=1732446941011; duration=0sec 2024-11-24T11:15:41,508 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/A, priority=13, startTime=1732446941010; duration=0sec 2024-11-24T11:15:41,508 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:41,508 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:C 2024-11-24T11:15:41,508 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:41,508 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:A 2024-11-24T11:15:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f79d0f35d64540378c2ef10b91ceadef is 50, key is test_row_0/B:col10/1732446939842/Put/seqid=0 2024-11-24T11:15:41,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742138_1314 (size=12301) 2024-11-24T11:15:41,944 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f79d0f35d64540378c2ef10b91ceadef 2024-11-24T11:15:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/322e659685244e5e9e1e8fc3fc3ce537 is 50, key is test_row_0/C:col10/1732446939842/Put/seqid=0 2024-11-24T11:15:41,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742139_1315 (size=12301) 2024-11-24T11:15:41,968 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/322e659685244e5e9e1e8fc3fc3ce537 2024-11-24T11:15:41,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/c1d47842d7054c1195e8f8dc47f1ed7a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/c1d47842d7054c1195e8f8dc47f1ed7a 2024-11-24T11:15:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T11:15:41,988 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/c1d47842d7054c1195e8f8dc47f1ed7a, entries=150, sequenceid=342, filesize=12.0 K 2024-11-24T11:15:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/f79d0f35d64540378c2ef10b91ceadef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f79d0f35d64540378c2ef10b91ceadef 2024-11-24T11:15:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:41,997 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f79d0f35d64540378c2ef10b91ceadef, entries=150, sequenceid=342, filesize=12.0 K 2024-11-24T11:15:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/322e659685244e5e9e1e8fc3fc3ce537 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/322e659685244e5e9e1e8fc3fc3ce537 2024-11-24T11:15:42,004 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/322e659685244e5e9e1e8fc3fc3ce537, entries=150, sequenceid=342, filesize=12.0 K 2024-11-24T11:15:42,005 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for 76c783a6d487b45ef515b468f5ea8d41 in 957ms, sequenceid=342, compaction requested=false 2024-11-24T11:15:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-24T11:15:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-24T11:15:42,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T11:15:42,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-24T11:15:42,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1860 sec 2024-11-24T11:15:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:42,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:42,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:42,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:42,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:42,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:42,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:42,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:42,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/cf11e96e4bdf46de829671f7505e6c1b is 50, key is test_row_0/A:col10/1732446942013/Put/seqid=0 2024-11-24T11:15:42,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742140_1316 (size=12301) 2024-11-24T11:15:42,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446942059, max timestamp=1732446942100Num uniques 859; 2024-11-24T11:15:42,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446942059, max timestamp=1732446942100Num uniques 859; 2024-11-24T11:15:42,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446942059, max timestamp=1732446942100Num uniques 859; 2024-11-24T11:15:42,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:42,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:42,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:42,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447002201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:42,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447002314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:42,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/cf11e96e4bdf46de829671f7505e6c1b 2024-11-24T11:15:42,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/63b378d55b614a58a7d551b36be9539a is 50, key is test_row_0/B:col10/1732446942013/Put/seqid=0 2024-11-24T11:15:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742141_1317 (size=12301) 2024-11-24T11:15:42,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447002523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37306 deadline: 1732447002554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:42,558 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 76c783a6d487b45ef515b468f5ea8d41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:42,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447002834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:42,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:42,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:42,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/63b378d55b614a58a7d551b36be9539a 2024-11-24T11:15:42,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/cd29596780ef4b10ad7d5ded23d3fadf is 50, key is test_row_0/C:col10/1732446942013/Put/seqid=0 2024-11-24T11:15:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742142_1318 (size=12301) 2024-11-24T11:15:43,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/cd29596780ef4b10ad7d5ded23d3fadf 2024-11-24T11:15:43,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/cf11e96e4bdf46de829671f7505e6c1b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/cf11e96e4bdf46de829671f7505e6c1b 2024-11-24T11:15:43,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/cf11e96e4bdf46de829671f7505e6c1b, entries=150, sequenceid=354, filesize=12.0 K 2024-11-24T11:15:43,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/63b378d55b614a58a7d551b36be9539a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/63b378d55b614a58a7d551b36be9539a 2024-11-24T11:15:43,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/63b378d55b614a58a7d551b36be9539a, entries=150, sequenceid=354, filesize=12.0 K 2024-11-24T11:15:43,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/cd29596780ef4b10ad7d5ded23d3fadf as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/cd29596780ef4b10ad7d5ded23d3fadf 2024-11-24T11:15:43,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/cd29596780ef4b10ad7d5ded23d3fadf, entries=150, sequenceid=354, filesize=12.0 K 2024-11-24T11:15:43,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 76c783a6d487b45ef515b468f5ea8d41 in 1253ms, sequenceid=354, compaction requested=true 2024-11-24T11:15:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:15:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:43,272 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:15:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:43,272 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c783a6d487b45ef515b468f5ea8d41:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:15:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:43,273 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:43,273 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/B is initiating minor compaction (all files) 2024-11-24T11:15:43,273 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:43,273 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/B in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:43,273 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/A is initiating minor compaction (all files) 2024-11-24T11:15:43,273 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/A in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:43,273 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ad095b879d6f4388a1c375129abf0cdb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f79d0f35d64540378c2ef10b91ceadef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/63b378d55b614a58a7d551b36be9539a] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=36.7 K 2024-11-24T11:15:43,273 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/4446c5f198f04f3ab0d6fc99bac2136d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/c1d47842d7054c1195e8f8dc47f1ed7a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/cf11e96e4bdf46de829671f7505e6c1b] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=36.7 K 2024-11-24T11:15:43,273 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting ad095b879d6f4388a1c375129abf0cdb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732446939623 2024-11-24T11:15:43,274 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4446c5f198f04f3ab0d6fc99bac2136d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732446939623 2024-11-24T11:15:43,274 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f79d0f35d64540378c2ef10b91ceadef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732446939820 2024-11-24T11:15:43,274 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d47842d7054c1195e8f8dc47f1ed7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732446939820 2024-11-24T11:15:43,274 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf11e96e4bdf46de829671f7505e6c1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732446941998 2024-11-24T11:15:43,274 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 63b378d55b614a58a7d551b36be9539a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732446941998 2024-11-24T11:15:43,281 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#A#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:43,281 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:43,282 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/01be97f4fcac4eec843a1ac9263c9e3c is 50, key is test_row_0/A:col10/1732446942013/Put/seqid=0 2024-11-24T11:15:43,282 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/89ab20e914b8476f9114afbff6a94bdc is 50, key is test_row_0/B:col10/1732446942013/Put/seqid=0 2024-11-24T11:15:43,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742143_1319 (size=13119) 2024-11-24T11:15:43,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742144_1320 (size=13119) 2024-11-24T11:15:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:43,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-24T11:15:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T11:15:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T11:15:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T11:15:43,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/fd8350bdbccf4c539ba5b83b6b424e10 is 50, key is test_row_0/A:col10/1732446942193/Put/seqid=0 2024-11-24T11:15:43,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742145_1321 (size=12301) 2024-11-24T11:15:43,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447003384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:43,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:43,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447003494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:43,699 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/89ab20e914b8476f9114afbff6a94bdc as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/89ab20e914b8476f9114afbff6a94bdc 2024-11-24T11:15:43,700 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/01be97f4fcac4eec843a1ac9263c9e3c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/01be97f4fcac4eec843a1ac9263c9e3c 2024-11-24T11:15:43,706 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/B of 76c783a6d487b45ef515b468f5ea8d41 into 89ab20e914b8476f9114afbff6a94bdc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:43,706 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:43,706 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/B, priority=13, startTime=1732446943272; duration=0sec 2024-11-24T11:15:43,706 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:15:43,706 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:B 2024-11-24T11:15:43,706 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:15:43,708 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/A of 76c783a6d487b45ef515b468f5ea8d41 into 01be97f4fcac4eec843a1ac9263c9e3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:43,708 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:43,708 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/A, priority=13, startTime=1732446943272; duration=0sec 2024-11-24T11:15:43,708 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:43,708 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:A 2024-11-24T11:15:43,708 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:15:43,709 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 76c783a6d487b45ef515b468f5ea8d41/C is initiating minor compaction (all files) 2024-11-24T11:15:43,709 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c783a6d487b45ef515b468f5ea8d41/C in TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:43,709 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/308f0e80ca474a45b35b38a9dc18d625, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/322e659685244e5e9e1e8fc3fc3ce537, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/cd29596780ef4b10ad7d5ded23d3fadf] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp, totalSize=36.7 K 2024-11-24T11:15:43,709 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 308f0e80ca474a45b35b38a9dc18d625, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732446939623 2024-11-24T11:15:43,710 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 322e659685244e5e9e1e8fc3fc3ce537, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732446939820 2024-11-24T11:15:43,710 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting cd29596780ef4b10ad7d5ded23d3fadf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732446941998 2024-11-24T11:15:43,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447003714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:43,723 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c783a6d487b45ef515b468f5ea8d41#C#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:15:43,723 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/08c889a0c9594e96b3464bf23ef45d4d is 50, key is test_row_0/C:col10/1732446942013/Put/seqid=0 2024-11-24T11:15:43,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742146_1322 (size=13119) 2024-11-24T11:15:43,733 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/08c889a0c9594e96b3464bf23ef45d4d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/08c889a0c9594e96b3464bf23ef45d4d 2024-11-24T11:15:43,738 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c783a6d487b45ef515b468f5ea8d41/C of 76c783a6d487b45ef515b468f5ea8d41 into 08c889a0c9594e96b3464bf23ef45d4d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:15:43,738 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:43,738 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., storeName=76c783a6d487b45ef515b468f5ea8d41/C, priority=13, startTime=1732446943272; duration=0sec 2024-11-24T11:15:43,738 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:15:43,739 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c783a6d487b45ef515b468f5ea8d41:C 2024-11-24T11:15:43,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/fd8350bdbccf4c539ba5b83b6b424e10 2024-11-24T11:15:43,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/e2dd583039614595800f8fa5d843323d is 50, key is test_row_0/B:col10/1732446942193/Put/seqid=0 2024-11-24T11:15:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742147_1323 (size=12301) 2024-11-24T11:15:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T11:15:43,994 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:43,995 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-24T11:15:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T11:15:43,998 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:44,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:44,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:44,026 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-11-24T11:15:44,026 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:44,026 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,026 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37308 deadline: 1732447004023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:44,030 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-24T11:15:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:15:44,030 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:44,030 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,030 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,030 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-24T11:15:44,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:44,031 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:44,031 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,031 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:44,031 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-11-24T11:15:44,031 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:44,031 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,031 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:44,035 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-24T11:15:44,035 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:44,035 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,035 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T11:15:44,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T11:15:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:44,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:44,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/e2dd583039614595800f8fa5d843323d 2024-11-24T11:15:44,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/888b2adfb59e4894bc615c223199cc35 is 50, key is test_row_0/C:col10/1732446942193/Put/seqid=0 2024-11-24T11:15:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742148_1324 (size=12301) 2024-11-24T11:15:44,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/888b2adfb59e4894bc615c223199cc35 2024-11-24T11:15:44,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/fd8350bdbccf4c539ba5b83b6b424e10 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/fd8350bdbccf4c539ba5b83b6b424e10 2024-11-24T11:15:44,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/fd8350bdbccf4c539ba5b83b6b424e10, entries=150, sequenceid=392, filesize=12.0 K 2024-11-24T11:15:44,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/e2dd583039614595800f8fa5d843323d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/e2dd583039614595800f8fa5d843323d 2024-11-24T11:15:44,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/e2dd583039614595800f8fa5d843323d, entries=150, sequenceid=392, filesize=12.0 K 2024-11-24T11:15:44,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/888b2adfb59e4894bc615c223199cc35 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/888b2adfb59e4894bc615c223199cc35 2024-11-24T11:15:44,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/888b2adfb59e4894bc615c223199cc35, entries=150, sequenceid=392, filesize=12.0 K 2024-11-24T11:15:44,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 76c783a6d487b45ef515b468f5ea8d41 in 880ms, sequenceid=392, compaction requested=false 2024-11-24T11:15:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:44,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T11:15:44,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:44,310 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T11:15:44,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:44,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:44,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:44,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:44,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:44,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/0a7ddc82226c4757b425e6225f20f4a2 is 50, key is test_row_1/A:col10/1732446943382/Put/seqid=0 2024-11-24T11:15:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T11:15:44,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742149_1325 (size=9857) 2024-11-24T11:15:44,519 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T11:15:44,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. as already flushing 2024-11-24T11:15:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:44,536 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-24T11:15:44,536 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:44,536 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,536 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:44,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T11:15:44,718 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/0a7ddc82226c4757b425e6225f20f4a2 2024-11-24T11:15:44,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/28fc53c848f547efbebd85278da92e47 is 50, key is test_row_1/B:col10/1732446943382/Put/seqid=0 2024-11-24T11:15:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742150_1326 (size=9857) 2024-11-24T11:15:45,127 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/28fc53c848f547efbebd85278da92e47 2024-11-24T11:15:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T11:15:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/1f201f5b0c2c495fad6effb97fe6b90e is 50, key is test_row_1/C:col10/1732446943382/Put/seqid=0 2024-11-24T11:15:45,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742151_1327 (size=9857) 2024-11-24T11:15:45,537 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/1f201f5b0c2c495fad6effb97fe6b90e 2024-11-24T11:15:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/0a7ddc82226c4757b425e6225f20f4a2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0a7ddc82226c4757b425e6225f20f4a2 2024-11-24T11:15:45,545 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0a7ddc82226c4757b425e6225f20f4a2, entries=100, sequenceid=404, filesize=9.6 K 2024-11-24T11:15:45,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/28fc53c848f547efbebd85278da92e47 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/28fc53c848f547efbebd85278da92e47 2024-11-24T11:15:45,549 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/28fc53c848f547efbebd85278da92e47, entries=100, sequenceid=404, filesize=9.6 K 2024-11-24T11:15:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/1f201f5b0c2c495fad6effb97fe6b90e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/1f201f5b0c2c495fad6effb97fe6b90e 2024-11-24T11:15:45,553 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/1f201f5b0c2c495fad6effb97fe6b90e, entries=100, sequenceid=404, filesize=9.6 K 2024-11-24T11:15:45,554 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for 76c783a6d487b45ef515b468f5ea8d41 in 1244ms, sequenceid=404, compaction requested=true 2024-11-24T11:15:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 76c783a6d487b45ef515b468f5ea8d41: 2024-11-24T11:15:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-24T11:15:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-24T11:15:45,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T11:15:45,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5560 sec 2024-11-24T11:15:45,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5620 sec 2024-11-24T11:15:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T11:15:46,154 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:46,605 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-24T11:15:46,605 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:46,605 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:46,605 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:46,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:50,395 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-24T11:15:50,395 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:50,395 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,395 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:50,445 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-24T11:15:50,445 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:50,445 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,445 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:50,515 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-24T11:15:50,515 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:15:50,515 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,516 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3170 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3138 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3162 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3162 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3129 2024-11-24T11:15:50,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T11:15:50,516 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:15:50,516 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:15:50,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:50,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:15:50,517 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-24T11:15:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T11:15:50,521 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446950520"}]},"ts":"1732446950520"} 2024-11-24T11:15:50,523 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T11:15:50,523 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T11:15:50,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:15:50,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, UNASSIGN}] 2024-11-24T11:15:50,527 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, UNASSIGN 2024-11-24T11:15:50,527 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=76c783a6d487b45ef515b468f5ea8d41, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:50,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, UNASSIGN because future has completed 2024-11-24T11:15:50,529 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:15:50,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:15:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T11:15:50,683 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:50,683 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:15:50,684 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 76c783a6d487b45ef515b468f5ea8d41, disabling compactions & flushes 2024-11-24T11:15:50,684 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:50,684 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:50,684 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. after waiting 0 ms 2024-11-24T11:15:50,684 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:50,684 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 76c783a6d487b45ef515b468f5ea8d41 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T11:15:50,685 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=A 2024-11-24T11:15:50,685 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:50,685 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=B 2024-11-24T11:15:50,685 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:50,685 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c783a6d487b45ef515b468f5ea8d41, store=C 2024-11-24T11:15:50,685 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:50,692 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/01ea2cdf7f834b6b8fe764d79111b23f is 50, key is test_row_0/A:col10/1732446950444/Put/seqid=0 2024-11-24T11:15:50,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742152_1328 (size=12301) 2024-11-24T11:15:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T11:15:51,099 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/01ea2cdf7f834b6b8fe764d79111b23f 2024-11-24T11:15:51,108 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/ef9f5d3ef2bf422b943f534ebdf5b19e is 50, key is test_row_0/B:col10/1732446950444/Put/seqid=0 2024-11-24T11:15:51,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742153_1329 (size=12301) 2024-11-24T11:15:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T11:15:51,513 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/ef9f5d3ef2bf422b943f534ebdf5b19e 2024-11-24T11:15:51,526 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/768b6cac9c4046c29e88d98a420568d6 is 50, key is test_row_0/C:col10/1732446950444/Put/seqid=0 2024-11-24T11:15:51,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742154_1330 (size=12301) 2024-11-24T11:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T11:15:51,931 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/768b6cac9c4046c29e88d98a420568d6 2024-11-24T11:15:51,935 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/A/01ea2cdf7f834b6b8fe764d79111b23f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/01ea2cdf7f834b6b8fe764d79111b23f 2024-11-24T11:15:51,938 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/01ea2cdf7f834b6b8fe764d79111b23f, entries=150, sequenceid=412, filesize=12.0 K 2024-11-24T11:15:51,939 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/B/ef9f5d3ef2bf422b943f534ebdf5b19e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ef9f5d3ef2bf422b943f534ebdf5b19e 2024-11-24T11:15:51,942 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ef9f5d3ef2bf422b943f534ebdf5b19e, entries=150, sequenceid=412, filesize=12.0 K 2024-11-24T11:15:51,942 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/.tmp/C/768b6cac9c4046c29e88d98a420568d6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/768b6cac9c4046c29e88d98a420568d6 2024-11-24T11:15:51,946 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/768b6cac9c4046c29e88d98a420568d6, entries=150, sequenceid=412, filesize=12.0 K 2024-11-24T11:15:51,946 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 76c783a6d487b45ef515b468f5ea8d41 in 1262ms, sequenceid=412, compaction requested=true 2024-11-24T11:15:51,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/ea70fe4fa82d49ec8dbca77f87ff8164, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/f1f6edaa1b0d4ff0a817e7783d682fe9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/92b331576e0747bd96bab332f55343c4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/389b511f3ec34c3aa6fd1ff9926cd065, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d2bf3211dc0a4ad8928c764cc1fd85d9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6d7f1a7749964b9d87bd2b38b450ae5d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d5acb9bf927444d7afa1c7812e9889e2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a755ee63e78341b2a292bdd292bb15e8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0d4973305911494fa1000faf75c6ad9f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/225450ae466248cbb3e0081575b671ee, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/2fc9ac5bbd6b4ef08076c84030dff649, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/9c95b7496367485a927840f8fa22fb54, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6745d3a78b6b4b72a671adbfc209ef58, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/8ea111dd047e4ac5b06ee392c4a76cfd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a86c39040b534b49aa558295658b5c7d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e25191d8a0a9471c9ab6c52491930f00, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/be4f37b0b5e442c7bc0a2ec43786960c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/5dfd46149db24704bee7a7654abdc22c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/7fb2275fccc44efb9f48bda79696072c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d02bf03e11d742ef90913cdc726f0c38, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e6f6764a32e4424ba9acfd0322f7c937, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/4446c5f198f04f3ab0d6fc99bac2136d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/c1d47842d7054c1195e8f8dc47f1ed7a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/cf11e96e4bdf46de829671f7505e6c1b] to archive 2024-11-24T11:15:51,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:15:51,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/ea70fe4fa82d49ec8dbca77f87ff8164 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/ea70fe4fa82d49ec8dbca77f87ff8164 2024-11-24T11:15:51,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/f1f6edaa1b0d4ff0a817e7783d682fe9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/f1f6edaa1b0d4ff0a817e7783d682fe9 2024-11-24T11:15:51,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/92b331576e0747bd96bab332f55343c4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/92b331576e0747bd96bab332f55343c4 2024-11-24T11:15:51,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/389b511f3ec34c3aa6fd1ff9926cd065 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/389b511f3ec34c3aa6fd1ff9926cd065 2024-11-24T11:15:51,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d2bf3211dc0a4ad8928c764cc1fd85d9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d2bf3211dc0a4ad8928c764cc1fd85d9 2024-11-24T11:15:51,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6d7f1a7749964b9d87bd2b38b450ae5d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6d7f1a7749964b9d87bd2b38b450ae5d 2024-11-24T11:15:51,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d5acb9bf927444d7afa1c7812e9889e2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d5acb9bf927444d7afa1c7812e9889e2 2024-11-24T11:15:51,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a755ee63e78341b2a292bdd292bb15e8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a755ee63e78341b2a292bdd292bb15e8 2024-11-24T11:15:51,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0d4973305911494fa1000faf75c6ad9f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0d4973305911494fa1000faf75c6ad9f 2024-11-24T11:15:51,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/225450ae466248cbb3e0081575b671ee to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/225450ae466248cbb3e0081575b671ee 2024-11-24T11:15:51,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/2fc9ac5bbd6b4ef08076c84030dff649 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/2fc9ac5bbd6b4ef08076c84030dff649 2024-11-24T11:15:51,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/9c95b7496367485a927840f8fa22fb54 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/9c95b7496367485a927840f8fa22fb54 2024-11-24T11:15:51,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6745d3a78b6b4b72a671adbfc209ef58 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/6745d3a78b6b4b72a671adbfc209ef58 2024-11-24T11:15:51,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/8ea111dd047e4ac5b06ee392c4a76cfd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/8ea111dd047e4ac5b06ee392c4a76cfd 2024-11-24T11:15:51,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a86c39040b534b49aa558295658b5c7d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/a86c39040b534b49aa558295658b5c7d 2024-11-24T11:15:51,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e25191d8a0a9471c9ab6c52491930f00 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e25191d8a0a9471c9ab6c52491930f00 2024-11-24T11:15:51,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/be4f37b0b5e442c7bc0a2ec43786960c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/be4f37b0b5e442c7bc0a2ec43786960c 2024-11-24T11:15:51,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/5dfd46149db24704bee7a7654abdc22c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/5dfd46149db24704bee7a7654abdc22c 2024-11-24T11:15:51,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/7fb2275fccc44efb9f48bda79696072c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/7fb2275fccc44efb9f48bda79696072c 2024-11-24T11:15:51,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d02bf03e11d742ef90913cdc726f0c38 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/d02bf03e11d742ef90913cdc726f0c38 2024-11-24T11:15:51,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e6f6764a32e4424ba9acfd0322f7c937 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/e6f6764a32e4424ba9acfd0322f7c937 2024-11-24T11:15:51,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/4446c5f198f04f3ab0d6fc99bac2136d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/4446c5f198f04f3ab0d6fc99bac2136d 2024-11-24T11:15:51,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/c1d47842d7054c1195e8f8dc47f1ed7a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/c1d47842d7054c1195e8f8dc47f1ed7a 2024-11-24T11:15:51,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/cf11e96e4bdf46de829671f7505e6c1b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/cf11e96e4bdf46de829671f7505e6c1b 2024-11-24T11:15:51,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f8f30f27b09441d3b99bddc769179ccb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2ded6648ced1405a9256c686294ecfba, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/c7087ce4a8c646d1bb0f0dd5f5c3e082, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/fed4164292ac4c76bf62036b798cbc01, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/b5c8a2b9e4184e429a8318c8117171a5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/d20af67afff94dfbaea9534ebb952a28, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/7bc72593100f4da0a9aa681b9f40a7f4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/da3ab028553641bfbf81579e9589282d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2e49fb6336f748c9b729e4b48e652133, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f721666d663449eabbed63342de13017, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/38709b91ba0e47898824f4019401c4ea, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/a71902dedd7e4fe49a55fec90d58bd65, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/5c449523823e44709be16300c099482f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/4340e746cf6e4cefb3befd6598e897c9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/726bd2b22e964631a48b5ba84781b994, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3a3169c2dd9c4866adcac78dc044d8b8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3756c282df7747cca90de4b3e2d240ab, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/25e2441aebb44be6a44db130c997d83a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/155f1fb955f14e7eb2768ba3ae4eb758, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/681ea80dda56481ebc50fa76a699a9e8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ad095b879d6f4388a1c375129abf0cdb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/bcfab545bd5c42a3900fde1a6654bc3f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f79d0f35d64540378c2ef10b91ceadef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/63b378d55b614a58a7d551b36be9539a] to archive 2024-11-24T11:15:51,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:15:51,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f8f30f27b09441d3b99bddc769179ccb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f8f30f27b09441d3b99bddc769179ccb 2024-11-24T11:15:51,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2ded6648ced1405a9256c686294ecfba to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2ded6648ced1405a9256c686294ecfba 2024-11-24T11:15:51,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/c7087ce4a8c646d1bb0f0dd5f5c3e082 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/c7087ce4a8c646d1bb0f0dd5f5c3e082 2024-11-24T11:15:51,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/fed4164292ac4c76bf62036b798cbc01 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/fed4164292ac4c76bf62036b798cbc01 2024-11-24T11:15:51,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/b5c8a2b9e4184e429a8318c8117171a5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/b5c8a2b9e4184e429a8318c8117171a5 2024-11-24T11:15:51,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/d20af67afff94dfbaea9534ebb952a28 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/d20af67afff94dfbaea9534ebb952a28 2024-11-24T11:15:51,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/7bc72593100f4da0a9aa681b9f40a7f4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/7bc72593100f4da0a9aa681b9f40a7f4 2024-11-24T11:15:51,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/da3ab028553641bfbf81579e9589282d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/da3ab028553641bfbf81579e9589282d 2024-11-24T11:15:51,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2e49fb6336f748c9b729e4b48e652133 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/2e49fb6336f748c9b729e4b48e652133 2024-11-24T11:15:51,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f721666d663449eabbed63342de13017 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f721666d663449eabbed63342de13017 2024-11-24T11:15:51,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/38709b91ba0e47898824f4019401c4ea to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/38709b91ba0e47898824f4019401c4ea 2024-11-24T11:15:51,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/a71902dedd7e4fe49a55fec90d58bd65 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/a71902dedd7e4fe49a55fec90d58bd65 2024-11-24T11:15:51,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/5c449523823e44709be16300c099482f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/5c449523823e44709be16300c099482f 2024-11-24T11:15:51,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/4340e746cf6e4cefb3befd6598e897c9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/4340e746cf6e4cefb3befd6598e897c9 2024-11-24T11:15:51,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/726bd2b22e964631a48b5ba84781b994 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/726bd2b22e964631a48b5ba84781b994 2024-11-24T11:15:51,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3a3169c2dd9c4866adcac78dc044d8b8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3a3169c2dd9c4866adcac78dc044d8b8 2024-11-24T11:15:51,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3756c282df7747cca90de4b3e2d240ab to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/3756c282df7747cca90de4b3e2d240ab 2024-11-24T11:15:51,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/25e2441aebb44be6a44db130c997d83a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/25e2441aebb44be6a44db130c997d83a 2024-11-24T11:15:51,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/155f1fb955f14e7eb2768ba3ae4eb758 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/155f1fb955f14e7eb2768ba3ae4eb758 2024-11-24T11:15:51,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/681ea80dda56481ebc50fa76a699a9e8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/681ea80dda56481ebc50fa76a699a9e8 2024-11-24T11:15:51,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ad095b879d6f4388a1c375129abf0cdb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ad095b879d6f4388a1c375129abf0cdb 2024-11-24T11:15:51,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/bcfab545bd5c42a3900fde1a6654bc3f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/bcfab545bd5c42a3900fde1a6654bc3f 2024-11-24T11:15:51,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f79d0f35d64540378c2ef10b91ceadef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/f79d0f35d64540378c2ef10b91ceadef 2024-11-24T11:15:51,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/63b378d55b614a58a7d551b36be9539a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/63b378d55b614a58a7d551b36be9539a 2024-11-24T11:15:51,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e6571c1e720f4064a359374b4110cbfc, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8a0e89bda1ee4a2bba86d536cfcf80ab, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/70a764736f0e45a3908613fe09fb0c31, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a15af08f970e4af4ab762f7fe5ce363b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e1c50b9c8d6542b0add6c4c3ddbe0a97, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8f7dfc7c3ea146a8a3a13c526ce1f39e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/72587eb13f8a49b1aff268b59737d42f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/49243d09315542d281b510fc08f5599d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/c139d74ddc5244898a436e8442900108, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/629fac6f05aa4ad6a2f72a7a7ccbeeb1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/435d0b27aa8743ab8a6493cbd1037452, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a044eb8dff444729b36b9f54dad3ea0d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e30d48d6cf1c48f2b4f2b437de9b2720, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/709649a836ee4c0e85c5c83fe83c5629, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/2ac5009118aa49e59b634b88fe3aeeaa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/87e76f08f6c240a1a09772974291e940, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b669e69a038f49c68e24d2e36d46f403, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/278fd0ea3bf74281a5f6a7aaed09a2aa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/79be35b36f1f4301978c0ae0ac56a821, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b84e28774b224d96b4fb74e6f369af5d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/308f0e80ca474a45b35b38a9dc18d625, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/95474c6d952b4e47b1df401e3d9a9648, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/322e659685244e5e9e1e8fc3fc3ce537, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/cd29596780ef4b10ad7d5ded23d3fadf] to archive 2024-11-24T11:15:52,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:15:52,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e6571c1e720f4064a359374b4110cbfc to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e6571c1e720f4064a359374b4110cbfc 2024-11-24T11:15:52,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8a0e89bda1ee4a2bba86d536cfcf80ab to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8a0e89bda1ee4a2bba86d536cfcf80ab 2024-11-24T11:15:52,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/70a764736f0e45a3908613fe09fb0c31 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/70a764736f0e45a3908613fe09fb0c31 2024-11-24T11:15:52,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a15af08f970e4af4ab762f7fe5ce363b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a15af08f970e4af4ab762f7fe5ce363b 2024-11-24T11:15:52,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e1c50b9c8d6542b0add6c4c3ddbe0a97 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e1c50b9c8d6542b0add6c4c3ddbe0a97 2024-11-24T11:15:52,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8f7dfc7c3ea146a8a3a13c526ce1f39e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/8f7dfc7c3ea146a8a3a13c526ce1f39e 2024-11-24T11:15:52,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/72587eb13f8a49b1aff268b59737d42f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/72587eb13f8a49b1aff268b59737d42f 2024-11-24T11:15:52,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/49243d09315542d281b510fc08f5599d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/49243d09315542d281b510fc08f5599d 2024-11-24T11:15:52,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/c139d74ddc5244898a436e8442900108 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/c139d74ddc5244898a436e8442900108 2024-11-24T11:15:52,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/629fac6f05aa4ad6a2f72a7a7ccbeeb1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/629fac6f05aa4ad6a2f72a7a7ccbeeb1 2024-11-24T11:15:52,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/435d0b27aa8743ab8a6493cbd1037452 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/435d0b27aa8743ab8a6493cbd1037452 2024-11-24T11:15:52,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a044eb8dff444729b36b9f54dad3ea0d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/a044eb8dff444729b36b9f54dad3ea0d 2024-11-24T11:15:52,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e30d48d6cf1c48f2b4f2b437de9b2720 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/e30d48d6cf1c48f2b4f2b437de9b2720 2024-11-24T11:15:52,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/709649a836ee4c0e85c5c83fe83c5629 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/709649a836ee4c0e85c5c83fe83c5629 2024-11-24T11:15:52,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/2ac5009118aa49e59b634b88fe3aeeaa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/2ac5009118aa49e59b634b88fe3aeeaa 2024-11-24T11:15:52,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/87e76f08f6c240a1a09772974291e940 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/87e76f08f6c240a1a09772974291e940 2024-11-24T11:15:52,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b669e69a038f49c68e24d2e36d46f403 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b669e69a038f49c68e24d2e36d46f403 2024-11-24T11:15:52,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/278fd0ea3bf74281a5f6a7aaed09a2aa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/278fd0ea3bf74281a5f6a7aaed09a2aa 2024-11-24T11:15:52,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/79be35b36f1f4301978c0ae0ac56a821 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/79be35b36f1f4301978c0ae0ac56a821 2024-11-24T11:15:52,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b84e28774b224d96b4fb74e6f369af5d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/b84e28774b224d96b4fb74e6f369af5d 2024-11-24T11:15:52,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/308f0e80ca474a45b35b38a9dc18d625 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/308f0e80ca474a45b35b38a9dc18d625 2024-11-24T11:15:52,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/95474c6d952b4e47b1df401e3d9a9648 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/95474c6d952b4e47b1df401e3d9a9648 2024-11-24T11:15:52,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/322e659685244e5e9e1e8fc3fc3ce537 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/322e659685244e5e9e1e8fc3fc3ce537 2024-11-24T11:15:52,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/cd29596780ef4b10ad7d5ded23d3fadf to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/cd29596780ef4b10ad7d5ded23d3fadf 2024-11-24T11:15:52,031 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/recovered.edits/415.seqid, newMaxSeqId=415, maxSeqId=1 2024-11-24T11:15:52,032 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. 2024-11-24T11:15:52,032 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 76c783a6d487b45ef515b468f5ea8d41: Waiting for close lock at 1732446950683Running coprocessor pre-close hooks at 1732446950683Disabling compacts and flushes for region at 1732446950683Disabling writes for close at 1732446950684 (+1 ms)Obtaining lock to block concurrent updates at 1732446950684Preparing flush snapshotting stores in 76c783a6d487b45ef515b468f5ea8d41 at 1732446950684Finished memstore snapshotting TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732446950685 (+1 ms)Flushing stores of TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41. at 1732446950686 (+1 ms)Flushing 76c783a6d487b45ef515b468f5ea8d41/A: creating writer at 1732446950686Flushing 76c783a6d487b45ef515b468f5ea8d41/A: appending metadata at 1732446950691 (+5 ms)Flushing 76c783a6d487b45ef515b468f5ea8d41/A: closing flushed file at 1732446950691Flushing 76c783a6d487b45ef515b468f5ea8d41/B: creating writer at 1732446951104 (+413 ms)Flushing 76c783a6d487b45ef515b468f5ea8d41/B: appending metadata at 1732446951107 (+3 ms)Flushing 76c783a6d487b45ef515b468f5ea8d41/B: closing flushed file at 1732446951107Flushing 76c783a6d487b45ef515b468f5ea8d41/C: creating writer at 1732446951521 (+414 ms)Flushing 76c783a6d487b45ef515b468f5ea8d41/C: appending metadata at 1732446951526 (+5 ms)Flushing 76c783a6d487b45ef515b468f5ea8d41/C: closing flushed file at 1732446951526Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@712a04bb: reopening flushed file at 1732446951935 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@116397bc: reopening flushed file at 1732446951938 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5de9cb0f: reopening flushed file at 1732446951942 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 76c783a6d487b45ef515b468f5ea8d41 in 1262ms, sequenceid=412, compaction requested=true at 1732446951946 (+4 ms)Writing region close event to WAL at 1732446952028 (+82 ms)Running coprocessor post-close hooks at 1732446952032 (+4 ms)Closed at 1732446952032 2024-11-24T11:15:52,034 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:52,035 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=76c783a6d487b45ef515b468f5ea8d41, regionState=CLOSED 2024-11-24T11:15:52,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:15:52,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T11:15:52,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 76c783a6d487b45ef515b468f5ea8d41, server=f5f73e76a839,45939,1732446856176 in 1.5080 sec 2024-11-24T11:15:52,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-24T11:15:52,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c783a6d487b45ef515b468f5ea8d41, UNASSIGN in 1.5140 sec 2024-11-24T11:15:52,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-24T11:15:52,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5180 sec 2024-11-24T11:15:52,044 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446952043"}]},"ts":"1732446952043"} 2024-11-24T11:15:52,045 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T11:15:52,045 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T11:15:52,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5290 sec 2024-11-24T11:15:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T11:15:52,665 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:52,667 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-24T11:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:52,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T11:15:52,674 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:52,676 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:52,679 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/recovered.edits] 2024-11-24T11:15:52,682 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/01be97f4fcac4eec843a1ac9263c9e3c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/01be97f4fcac4eec843a1ac9263c9e3c 2024-11-24T11:15:52,683 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/01ea2cdf7f834b6b8fe764d79111b23f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/01ea2cdf7f834b6b8fe764d79111b23f 2024-11-24T11:15:52,685 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0a7ddc82226c4757b425e6225f20f4a2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/0a7ddc82226c4757b425e6225f20f4a2 2024-11-24T11:15:52,686 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/fd8350bdbccf4c539ba5b83b6b424e10 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/A/fd8350bdbccf4c539ba5b83b6b424e10 2024-11-24T11:15:52,688 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/28fc53c848f547efbebd85278da92e47 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/28fc53c848f547efbebd85278da92e47 2024-11-24T11:15:52,689 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/89ab20e914b8476f9114afbff6a94bdc to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/89ab20e914b8476f9114afbff6a94bdc 2024-11-24T11:15:52,690 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/e2dd583039614595800f8fa5d843323d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/e2dd583039614595800f8fa5d843323d 2024-11-24T11:15:52,691 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ef9f5d3ef2bf422b943f534ebdf5b19e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/B/ef9f5d3ef2bf422b943f534ebdf5b19e 2024-11-24T11:15:52,693 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/08c889a0c9594e96b3464bf23ef45d4d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/08c889a0c9594e96b3464bf23ef45d4d 2024-11-24T11:15:52,694 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/1f201f5b0c2c495fad6effb97fe6b90e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/1f201f5b0c2c495fad6effb97fe6b90e 2024-11-24T11:15:52,695 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/768b6cac9c4046c29e88d98a420568d6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/768b6cac9c4046c29e88d98a420568d6 2024-11-24T11:15:52,696 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/888b2adfb59e4894bc615c223199cc35 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/C/888b2adfb59e4894bc615c223199cc35 2024-11-24T11:15:52,698 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/recovered.edits/415.seqid to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41/recovered.edits/415.seqid 2024-11-24T11:15:52,699 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/76c783a6d487b45ef515b468f5ea8d41 2024-11-24T11:15:52,699 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T11:15:52,701 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:52,704 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T11:15:52,706 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T11:15:52,707 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:52,707 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T11:15:52,707 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732446952707"}]},"ts":"9223372036854775807"} 2024-11-24T11:15:52,708 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T11:15:52,708 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 76c783a6d487b45ef515b468f5ea8d41, NAME => 'TestAcidGuarantees,,1732446921812.76c783a6d487b45ef515b468f5ea8d41.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T11:15:52,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T11:15:52,709 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732446952708"}]},"ts":"9223372036854775807"} 2024-11-24T11:15:52,710 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T11:15:52,710 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:52,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-24T11:15:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T11:15:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T11:15:52,784 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:52,790 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=350 (was 394), ProcessCount=11 (was 11), AvailableMemoryMB=6850 (was 5993) - AvailableMemoryMB LEAK? - 2024-11-24T11:15:52,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=350, ProcessCount=11, AvailableMemoryMB=6849 2024-11-24T11:15:52,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:15:52,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:15:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:52,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T11:15:52,801 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:52,801 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-24T11:15:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-24T11:15:52,802 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T11:15:52,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742155_1331 (size=1061) 2024-11-24T11:15:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-24T11:15:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-24T11:15:53,209 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:15:53,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742156_1332 (size=53) 2024-11-24T11:15:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-24T11:15:53,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:15:53,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f5f0024ce88b8047201380866bbadec0, disabling compactions & flushes 2024-11-24T11:15:53,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. after waiting 0 ms 2024-11-24T11:15:53,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,619 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,619 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f5f0024ce88b8047201380866bbadec0: Waiting for close lock at 1732446953618Disabling compacts and flushes for region at 1732446953618Disabling writes for close at 1732446953618Writing region close event to WAL at 1732446953619 (+1 ms)Closed at 1732446953619 2024-11-24T11:15:53,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T11:15:53,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732446953621"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732446953621"}]},"ts":"1732446953621"} 2024-11-24T11:15:53,624 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T11:15:53,626 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T11:15:53,626 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446953626"}]},"ts":"1732446953626"} 2024-11-24T11:15:53,628 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T11:15:53,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, ASSIGN}] 2024-11-24T11:15:53,629 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, ASSIGN 2024-11-24T11:15:53,630 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, ASSIGN; state=OFFLINE, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=false 2024-11-24T11:15:53,781 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:53,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, ASSIGN because future has completed 2024-11-24T11:15:53,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:15:53,941 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,941 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:15:53,941 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,941 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:15:53,942 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,942 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,943 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-24T11:15:53,945 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:53,946 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f0024ce88b8047201380866bbadec0 columnFamilyName A 2024-11-24T11:15:53,946 DEBUG [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:53,947 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(327): Store=f5f0024ce88b8047201380866bbadec0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:53,947 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,949 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:53,950 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f0024ce88b8047201380866bbadec0 columnFamilyName B 2024-11-24T11:15:53,950 DEBUG [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:53,950 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(327): Store=f5f0024ce88b8047201380866bbadec0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:53,951 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,952 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:53,952 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f0024ce88b8047201380866bbadec0 columnFamilyName C 2024-11-24T11:15:53,952 DEBUG [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:53,953 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(327): Store=f5f0024ce88b8047201380866bbadec0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:53,953 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,953 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,954 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,954 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,955 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,955 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,956 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:15:53,957 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,959 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:15:53,959 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened f5f0024ce88b8047201380866bbadec0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65452987, jitterRate=-0.024674490094184875}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:15:53,959 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:53,960 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for f5f0024ce88b8047201380866bbadec0: Running coprocessor pre-open hook at 1732446953942Writing region info on filesystem at 1732446953942Initializing all the Stores at 1732446953943 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446953943Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446953943Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446953943Cleaning up temporary data from old regions at 1732446953955 (+12 ms)Running coprocessor post-open hooks at 1732446953960 (+5 ms)Region opened successfully at 1732446953960 2024-11-24T11:15:53,961 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., pid=81, masterSystemTime=1732446953937 2024-11-24T11:15:53,963 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,964 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:53,964 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=OPEN, openSeqNum=2, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:53,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:15:53,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T11:15:53,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 in 185 msec 2024-11-24T11:15:53,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-24T11:15:53,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, ASSIGN in 342 msec 2024-11-24T11:15:53,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T11:15:53,974 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446953974"}]},"ts":"1732446953974"} 2024-11-24T11:15:53,976 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T11:15:53,978 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T11:15:53,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-24T11:15:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-24T11:15:54,964 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:54,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7789cf32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:54,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:54,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:54,967 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3836c6c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:54,970 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecf68fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:54,971 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:54,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:54,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T11:15:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f5f73e76a839,40345,1732446855400 2024-11-24T11:15:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e1dbf58 2024-11-24T11:15:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T11:15:54,976 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T11:15:54,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:15:54,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:15:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T11:15:54,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742157_1333 (size=1097) 2024-11-24T11:15:55,391 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T11:15:55,391 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T11:15:55,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:15:55,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, REOPEN/MOVE}] 2024-11-24T11:15:55,398 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, REOPEN/MOVE 2024-11-24T11:15:55,399 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:55,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, REOPEN/MOVE because future has completed 2024-11-24T11:15:55,401 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:15:55,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:15:55,554 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,554 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:15:55,554 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing f5f0024ce88b8047201380866bbadec0, disabling compactions & flushes 2024-11-24T11:15:55,554 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,554 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,554 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. after waiting 0 ms 2024-11-24T11:15:55,554 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,558 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T11:15:55,558 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,558 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for f5f0024ce88b8047201380866bbadec0: Waiting for close lock at 1732446955554Running coprocessor pre-close hooks at 1732446955554Disabling compacts and flushes for region at 1732446955554Disabling writes for close at 1732446955554Writing region close event to WAL at 1732446955555 (+1 ms)Running coprocessor post-close hooks at 1732446955558 (+3 ms)Closed at 1732446955558 2024-11-24T11:15:55,558 WARN [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: f5f0024ce88b8047201380866bbadec0 to self. 2024-11-24T11:15:55,560 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,560 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=CLOSED 2024-11-24T11:15:55,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:15:55,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-24T11:15:55,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 in 161 msec 2024-11-24T11:15:55,564 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, REOPEN/MOVE; state=CLOSED, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=true 2024-11-24T11:15:55,714 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:55,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, REOPEN/MOVE because future has completed 2024-11-24T11:15:55,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:15:55,749 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T11:15:55,875 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,875 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:15:55,876 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,876 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:15:55,876 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,876 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,879 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,881 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:55,881 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f0024ce88b8047201380866bbadec0 columnFamilyName A 2024-11-24T11:15:55,883 DEBUG [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:55,884 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(327): Store=f5f0024ce88b8047201380866bbadec0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:55,884 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,885 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:55,886 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f0024ce88b8047201380866bbadec0 columnFamilyName B 2024-11-24T11:15:55,886 DEBUG [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:55,886 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(327): Store=f5f0024ce88b8047201380866bbadec0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:55,887 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,887 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:15:55,887 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f0024ce88b8047201380866bbadec0 columnFamilyName C 2024-11-24T11:15:55,887 DEBUG [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:55,888 INFO [StoreOpener-f5f0024ce88b8047201380866bbadec0-1 {}] regionserver.HStore(327): Store=f5f0024ce88b8047201380866bbadec0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:15:55,888 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,888 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,889 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,890 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,891 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,891 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,892 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:15:55,893 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,894 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened f5f0024ce88b8047201380866bbadec0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73178333, jitterRate=0.09044213593006134}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:15:55,894 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,895 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for f5f0024ce88b8047201380866bbadec0: Running coprocessor pre-open hook at 1732446955877Writing region info on filesystem at 1732446955877Initializing all the Stores at 1732446955878 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446955878Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446955879 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446955879Cleaning up temporary data from old regions at 1732446955891 (+12 ms)Running coprocessor post-open hooks at 1732446955894 (+3 ms)Region opened successfully at 1732446955895 (+1 ms) 2024-11-24T11:15:55,895 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., pid=86, masterSystemTime=1732446955869 2024-11-24T11:15:55,897 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,897 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:55,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=OPEN, openSeqNum=5, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:55,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:15:55,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-24T11:15:55,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 in 182 msec 2024-11-24T11:15:55,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-24T11:15:55,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, REOPEN/MOVE in 503 msec 2024-11-24T11:15:55,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T11:15:55,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-24T11:15:55,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-24T11:15:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T11:15:55,908 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:15:55,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a884f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36436c63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,910 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4d6a28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f86ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,912 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b28cd2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4abc844a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f0df0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,915 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a56f5e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,916 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f78c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@927442d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77812610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e365ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@725928f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472b0ad7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,922 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df0142b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227a416f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,923 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ece0a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,925 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61950eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580773ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,926 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356f178a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,928 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfeeeb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34dcef37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,929 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132e53c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c9d386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5713b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac20fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,934 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435fc34f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2483a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:15:55,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:15:55,937 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:15:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:15:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d2c6cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:15:55,939 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:15:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d40a489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:15:55,940 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,940 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,940 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,941 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,941 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:55,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:15:55,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:15:55,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:55,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:15:55,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:55,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:15:55,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:15:55,956 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,961 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,964 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,966 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:15:55,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-24T11:15:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T11:15:55,969 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:15:55,969 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:15:55,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:15:55,976 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,979 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:15:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:15:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:15:55,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:15:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:15:56,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248489737f58cb403292d0f528022ee3f0_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446955952/Put/seqid=0 2024-11-24T11:15:56,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447016012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447016013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447016013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446956002, max timestamp=1732446956021Num uniques 859; 2024-11-24T11:15:56,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446956002, max timestamp=1732446956021Num uniques 859; 2024-11-24T11:15:56,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446956002, max timestamp=1732446956021Num uniques 859; 2024-11-24T11:15:56,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:56,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:56,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742158_1334 (size=12154) 2024-11-24T11:15:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T11:15:56,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:56,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:56,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447016133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447016133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447016134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447016138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447016139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447016243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447016253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447016263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:56,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:56,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T11:15:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447016344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447016344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,430 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:56,434 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248489737f58cb403292d0f528022ee3f0_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248489737f58cb403292d0f528022ee3f0_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:56,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/42d82f541bcc4f868f47e85ab2c0efbd, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:15:56,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/42d82f541bcc4f868f47e85ab2c0efbd is 175, key is test_row_0/A:col10/1732446955952/Put/seqid=0 2024-11-24T11:15:56,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742159_1335 (size=30955) 2024-11-24T11:15:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447016464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447016464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447016473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T11:15:56,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447016664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447016664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447016774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447016774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447016793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:56,839 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/42d82f541bcc4f868f47e85ab2c0efbd 2024-11-24T11:15:56,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/44d62e8610bc447fad0f5b57940d81cd is 50, key is test_row_0/B:col10/1732446955952/Put/seqid=0 2024-11-24T11:15:56,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742160_1336 (size=12001) 2024-11-24T11:15:56,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:56,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T11:15:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447017183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447017183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:57,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/44d62e8610bc447fad0f5b57940d81cd 2024-11-24T11:15:57,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/7806a2884cb94dbd9736dfe2a08ac8e0 is 50, key is test_row_0/C:col10/1732446955952/Put/seqid=0 2024-11-24T11:15:57,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742161_1337 (size=12001) 2024-11-24T11:15:57,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447017293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447017293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447017304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:57,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:57,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:57,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,564 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T11:15:57,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:15:57,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/7806a2884cb94dbd9736dfe2a08ac8e0 2024-11-24T11:15:57,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/42d82f541bcc4f868f47e85ab2c0efbd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/42d82f541bcc4f868f47e85ab2c0efbd 2024-11-24T11:15:57,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/42d82f541bcc4f868f47e85ab2c0efbd, entries=150, sequenceid=15, filesize=30.2 K 2024-11-24T11:15:57,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/44d62e8610bc447fad0f5b57940d81cd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/44d62e8610bc447fad0f5b57940d81cd 2024-11-24T11:15:57,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/44d62e8610bc447fad0f5b57940d81cd, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T11:15:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/7806a2884cb94dbd9736dfe2a08ac8e0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/7806a2884cb94dbd9736dfe2a08ac8e0 2024-11-24T11:15:57,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/7806a2884cb94dbd9736dfe2a08ac8e0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T11:15:57,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f5f0024ce88b8047201380866bbadec0 in 1756ms, sequenceid=15, compaction requested=false 2024-11-24T11:15:57,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:15:57,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-24T11:15:57,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:57,806 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T11:15:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:15:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:15:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:15:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:15:57,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124828f5db87d8a4069b0f18842e4d1bd0d_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446956137/Put/seqid=0 2024-11-24T11:15:57,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742162_1338 (size=12154) 2024-11-24T11:15:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T11:15:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:58,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:15:58,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:15:58,226 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124828f5db87d8a4069b0f18842e4d1bd0d_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124828f5db87d8a4069b0f18842e4d1bd0d_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:15:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/1b8e4c710dde46eebb81596a2961e853, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:15:58,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/1b8e4c710dde46eebb81596a2961e853 is 175, key is test_row_0/A:col10/1732446956137/Put/seqid=0 2024-11-24T11:15:58,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742163_1339 (size=30955) 2024-11-24T11:15:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447018240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447018247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447018313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447018314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,317 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in f5f0024ce88b8047201380866bbadec0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,317 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in f5f0024ce88b8047201380866bbadec0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447018324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,329 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in f5f0024ce88b8047201380866bbadec0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:15:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447018363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447018363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447018573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447018573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,641 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/1b8e4c710dde46eebb81596a2961e853 2024-11-24T11:15:58,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1d99245b2f114a9998a92babbc34aef4 is 50, key is test_row_0/B:col10/1732446956137/Put/seqid=0 2024-11-24T11:15:58,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742164_1340 (size=12001) 2024-11-24T11:15:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447018883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447018893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:58,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:58,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:59,058 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1d99245b2f114a9998a92babbc34aef4 2024-11-24T11:15:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/5147aa8921024717b0f15529a18d2125 is 50, key is test_row_0/C:col10/1732446956137/Put/seqid=0 2024-11-24T11:15:59,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742165_1341 (size=12001) 2024-11-24T11:15:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447019393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:15:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447019414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:15:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:15:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:15:59,471 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/5147aa8921024717b0f15529a18d2125 2024-11-24T11:15:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/1b8e4c710dde46eebb81596a2961e853 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/1b8e4c710dde46eebb81596a2961e853 2024-11-24T11:15:59,480 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/1b8e4c710dde46eebb81596a2961e853, entries=150, sequenceid=51, filesize=30.2 K 2024-11-24T11:15:59,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1d99245b2f114a9998a92babbc34aef4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1d99245b2f114a9998a92babbc34aef4 2024-11-24T11:15:59,484 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1d99245b2f114a9998a92babbc34aef4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T11:15:59,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/5147aa8921024717b0f15529a18d2125 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/5147aa8921024717b0f15529a18d2125 2024-11-24T11:15:59,488 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/5147aa8921024717b0f15529a18d2125, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T11:15:59,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for f5f0024ce88b8047201380866bbadec0 in 1684ms, sequenceid=51, compaction requested=false 2024-11-24T11:15:59,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:15:59,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:15:59,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-24T11:15:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-24T11:15:59,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-24T11:15:59,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5200 sec 2024-11-24T11:15:59,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.5270 sec 2024-11-24T11:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T11:16:00,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:00,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-24T11:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T11:16:00,136 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:00,137 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:00,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T11:16:00,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-24T11:16:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:00,289 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T11:16:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:00,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:00,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:00,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124eb261c875ced43acb308222cf0086cd1_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446958238/Put/seqid=0 2024-11-24T11:16:00,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742166_1342 (size=12154) 2024-11-24T11:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:00,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:00,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447020379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447020380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446960365, max timestamp=1732446960383Num uniques 859; 2024-11-24T11:16:00,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446960365, max timestamp=1732446960383Num uniques 859; 2024-11-24T11:16:00,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732446960365, max timestamp=1732446960383Num uniques 859; 2024-11-24T11:16:00,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:16:00,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:16:00,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T11:16:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T11:16:00,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447020452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447020453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447020454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447020494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447020494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447020573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447020574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447020574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:00,702 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124eb261c875ced43acb308222cf0086cd1_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124eb261c875ced43acb308222cf0086cd1_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:00,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/38ffb2508ab041808cd81b6683617c79, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/38ffb2508ab041808cd81b6683617c79 is 175, key is test_row_0/A:col10/1732446958238/Put/seqid=0 2024-11-24T11:16:00,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742167_1343 (size=30955) 2024-11-24T11:16:00,710 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/38ffb2508ab041808cd81b6683617c79 2024-11-24T11:16:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4b992afb263a4cbe86f09458aa6ce39f is 50, key is test_row_0/B:col10/1732446958238/Put/seqid=0 2024-11-24T11:16:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742168_1344 (size=12001) 2024-11-24T11:16:00,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447020714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447020714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T11:16:00,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447020784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447020784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447020793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447021033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447021033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447021104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447021104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,119 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4b992afb263a4cbe86f09458aa6ce39f 2024-11-24T11:16:01,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447021113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d6dea9b2c18a4b359ac5124f11f0a553 is 50, key is test_row_0/C:col10/1732446958238/Put/seqid=0 2024-11-24T11:16:01,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742169_1345 (size=12001) 2024-11-24T11:16:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T11:16:01,529 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d6dea9b2c18a4b359ac5124f11f0a553 2024-11-24T11:16:01,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447021553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447021553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/38ffb2508ab041808cd81b6683617c79 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/38ffb2508ab041808cd81b6683617c79 2024-11-24T11:16:01,584 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/38ffb2508ab041808cd81b6683617c79, entries=150, sequenceid=62, filesize=30.2 K 2024-11-24T11:16:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4b992afb263a4cbe86f09458aa6ce39f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4b992afb263a4cbe86f09458aa6ce39f 2024-11-24T11:16:01,588 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4b992afb263a4cbe86f09458aa6ce39f, entries=150, sequenceid=62, filesize=11.7 K 2024-11-24T11:16:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d6dea9b2c18a4b359ac5124f11f0a553 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d6dea9b2c18a4b359ac5124f11f0a553 2024-11-24T11:16:01,593 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d6dea9b2c18a4b359ac5124f11f0a553, entries=150, sequenceid=62, filesize=11.7 K 2024-11-24T11:16:01,594 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f5f0024ce88b8047201380866bbadec0 in 1305ms, sequenceid=62, compaction requested=true 2024-11-24T11:16:01,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:01,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:01,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-24T11:16:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-24T11:16:01,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-24T11:16:01,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-11-24T11:16:01,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.4630 sec 2024-11-24T11:16:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:01,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T11:16:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:16:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:16:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:16:01,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242a03e879a864472281a5274f6fa3db3b_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446960435/Put/seqid=0 2024-11-24T11:16:01,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742170_1346 (size=14594) 2024-11-24T11:16:01,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447021646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447021649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447021654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447021764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447021764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447021773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447021974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447021974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447021993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,036 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:02,039 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242a03e879a864472281a5274f6fa3db3b_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242a03e879a864472281a5274f6fa3db3b_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:02,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/756983697f9c47e6b35ba6eaa944112b, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:02,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/756983697f9c47e6b35ba6eaa944112b is 175, key is test_row_0/A:col10/1732446960435/Put/seqid=0 2024-11-24T11:16:02,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742171_1347 (size=39549) 2024-11-24T11:16:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T11:16:02,284 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:02,285 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-24T11:16:02,287 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:02,289 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:02,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T11:16:02,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447022294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447022294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447022314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T11:16:02,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-24T11:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/756983697f9c47e6b35ba6eaa944112b 2024-11-24T11:16:02,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/9a3f328ebc1b4a1aa32d6be217a14bfa is 50, key is test_row_0/B:col10/1732446960435/Put/seqid=0 2024-11-24T11:16:02,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742172_1348 (size=12001) 2024-11-24T11:16:02,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/9a3f328ebc1b4a1aa32d6be217a14bfa 2024-11-24T11:16:02,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/26415018d72e473da0b3f9d60d4bd8f2 is 50, key is test_row_0/C:col10/1732446960435/Put/seqid=0 2024-11-24T11:16:02,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742173_1349 (size=12001) 2024-11-24T11:16:02,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/26415018d72e473da0b3f9d60d4bd8f2 2024-11-24T11:16:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/756983697f9c47e6b35ba6eaa944112b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/756983697f9c47e6b35ba6eaa944112b 2024-11-24T11:16:02,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/756983697f9c47e6b35ba6eaa944112b, entries=200, sequenceid=97, filesize=38.6 K 2024-11-24T11:16:02,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/9a3f328ebc1b4a1aa32d6be217a14bfa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/9a3f328ebc1b4a1aa32d6be217a14bfa 2024-11-24T11:16:02,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/9a3f328ebc1b4a1aa32d6be217a14bfa, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T11:16:02,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/26415018d72e473da0b3f9d60d4bd8f2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/26415018d72e473da0b3f9d60d4bd8f2 2024-11-24T11:16:02,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/26415018d72e473da0b3f9d60d4bd8f2, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T11:16:02,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for f5f0024ce88b8047201380866bbadec0 in 945ms, sequenceid=97, compaction requested=true 2024-11-24T11:16:02,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:02,572 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:16:02,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:02,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:02,572 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:16:02,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:02,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:02,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:02,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:02,573 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:16:02,573 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/A is initiating minor compaction (all files) 2024-11-24T11:16:02,573 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/A in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,573 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/42d82f541bcc4f868f47e85ab2c0efbd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/1b8e4c710dde46eebb81596a2961e853, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/38ffb2508ab041808cd81b6683617c79, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/756983697f9c47e6b35ba6eaa944112b] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=129.3 K 2024-11-24T11:16:02,573 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,573 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/42d82f541bcc4f868f47e85ab2c0efbd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/1b8e4c710dde46eebb81596a2961e853, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/38ffb2508ab041808cd81b6683617c79, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/756983697f9c47e6b35ba6eaa944112b] 2024-11-24T11:16:02,573 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:16:02,573 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/B is initiating minor compaction (all files) 2024-11-24T11:16:02,574 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d82f541bcc4f868f47e85ab2c0efbd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732446955951 2024-11-24T11:16:02,574 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/B in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,574 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/44d62e8610bc447fad0f5b57940d81cd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1d99245b2f114a9998a92babbc34aef4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4b992afb263a4cbe86f09458aa6ce39f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/9a3f328ebc1b4a1aa32d6be217a14bfa] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=46.9 K 2024-11-24T11:16:02,574 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b8e4c710dde46eebb81596a2961e853, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732446956121 2024-11-24T11:16:02,574 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 44d62e8610bc447fad0f5b57940d81cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732446955951 2024-11-24T11:16:02,574 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ffb2508ab041808cd81b6683617c79, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446958222 2024-11-24T11:16:02,574 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d99245b2f114a9998a92babbc34aef4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732446956121 2024-11-24T11:16:02,575 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756983697f9c47e6b35ba6eaa944112b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732446960435 2024-11-24T11:16:02,575 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b992afb263a4cbe86f09458aa6ce39f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446958222 2024-11-24T11:16:02,575 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a3f328ebc1b4a1aa32d6be217a14bfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732446960435 2024-11-24T11:16:02,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:02,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:02,593 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:02,596 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:02,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-24T11:16:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,597 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/e6b4668c9d4c4587b9352607dfccca08 is 50, key is test_row_0/B:col10/1732446960435/Put/seqid=0 2024-11-24T11:16:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T11:16:02,607 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124245fc72e3259476daf5e7fc8c5f7748a_f5f0024ce88b8047201380866bbadec0 store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:02,608 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124245fc72e3259476daf5e7fc8c5f7748a_f5f0024ce88b8047201380866bbadec0, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:02,609 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124245fc72e3259476daf5e7fc8c5f7748a_f5f0024ce88b8047201380866bbadec0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:02,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f6797dc7b6fb4d3baa87f555da32a3aa_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446961626/Put/seqid=0 2024-11-24T11:16:02,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742174_1350 (size=12139) 2024-11-24T11:16:02,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742175_1351 (size=4469) 2024-11-24T11:16:02,638 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#A#compaction#297 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:02,639 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/4ded5e298d3045169fabbfffaae11f20 is 175, key is test_row_0/A:col10/1732446960435/Put/seqid=0 2024-11-24T11:16:02,641 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/e6b4668c9d4c4587b9352607dfccca08 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/e6b4668c9d4c4587b9352607dfccca08 2024-11-24T11:16:02,645 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5f0024ce88b8047201380866bbadec0/B of f5f0024ce88b8047201380866bbadec0 into e6b4668c9d4c4587b9352607dfccca08(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:02,645 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:02,645 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/B, priority=12, startTime=1732446962572; duration=0sec 2024-11-24T11:16:02,645 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:02,645 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:B 2024-11-24T11:16:02,645 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:16:02,647 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:16:02,647 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/C is initiating minor compaction (all files) 2024-11-24T11:16:02,647 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/C in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,647 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/7806a2884cb94dbd9736dfe2a08ac8e0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/5147aa8921024717b0f15529a18d2125, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d6dea9b2c18a4b359ac5124f11f0a553, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/26415018d72e473da0b3f9d60d4bd8f2] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=46.9 K 2024-11-24T11:16:02,647 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7806a2884cb94dbd9736dfe2a08ac8e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732446955951 2024-11-24T11:16:02,648 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 5147aa8921024717b0f15529a18d2125, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732446956121 2024-11-24T11:16:02,648 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d6dea9b2c18a4b359ac5124f11f0a553, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446958222 2024-11-24T11:16:02,648 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 26415018d72e473da0b3f9d60d4bd8f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732446960435 2024-11-24T11:16:02,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742176_1352 (size=19474) 2024-11-24T11:16:02,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:02,666 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f6797dc7b6fb4d3baa87f555da32a3aa_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f6797dc7b6fb4d3baa87f555da32a3aa_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:02,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f720290d2a1549cd9e06d754a6eae53e, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:02,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f720290d2a1549cd9e06d754a6eae53e is 175, key is test_row_0/A:col10/1732446961626/Put/seqid=0 2024-11-24T11:16:02,670 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:02,671 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/14c21458d18a4e9a817923f7d06cfccf is 50, key is test_row_0/C:col10/1732446960435/Put/seqid=0 2024-11-24T11:16:02,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742177_1353 (size=31093) 2024-11-24T11:16:02,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742178_1354 (size=56733) 2024-11-24T11:16:02,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742179_1355 (size=12139) 2024-11-24T11:16:02,704 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/4ded5e298d3045169fabbfffaae11f20 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/4ded5e298d3045169fabbfffaae11f20 2024-11-24T11:16:02,712 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/14c21458d18a4e9a817923f7d06cfccf as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/14c21458d18a4e9a817923f7d06cfccf 2024-11-24T11:16:02,719 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5f0024ce88b8047201380866bbadec0/C of f5f0024ce88b8047201380866bbadec0 into 14c21458d18a4e9a817923f7d06cfccf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:02,719 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:02,719 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/C, priority=12, startTime=1732446962573; duration=0sec 2024-11-24T11:16:02,719 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:02,719 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:C 2024-11-24T11:16:02,734 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5f0024ce88b8047201380866bbadec0/A of f5f0024ce88b8047201380866bbadec0 into 4ded5e298d3045169fabbfffaae11f20(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:02,734 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:02,734 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/A, priority=12, startTime=1732446962571; duration=0sec 2024-11-24T11:16:02,734 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:02,734 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:A 2024-11-24T11:16:02,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-24T11:16:02,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:02,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447022737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447022747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447022814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447022814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447022833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447022865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447022865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:02,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-24T11:16:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:02,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T11:16:03,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-24T11:16:03,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:03,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:03,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:03,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:03,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:03,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447023074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447023074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f720290d2a1549cd9e06d754a6eae53e 2024-11-24T11:16:03,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/5ee454898eb04a76bb8dc78792618640 is 50, key is test_row_0/B:col10/1732446961626/Put/seqid=0 2024-11-24T11:16:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742180_1356 (size=12001) 2024-11-24T11:16:03,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/5ee454898eb04a76bb8dc78792618640 2024-11-24T11:16:03,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d521e53077924b08b576e628fb202a2a is 50, key is test_row_0/C:col10/1732446961626/Put/seqid=0 2024-11-24T11:16:03,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742181_1357 (size=12001) 2024-11-24T11:16:03,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-24T11:16:03,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:03,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:03,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:03,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:03,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d521e53077924b08b576e628fb202a2a 2024-11-24T11:16:03,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f720290d2a1549cd9e06d754a6eae53e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f720290d2a1549cd9e06d754a6eae53e 2024-11-24T11:16:03,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f720290d2a1549cd9e06d754a6eae53e, entries=300, sequenceid=110, filesize=55.4 K 2024-11-24T11:16:03,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/5ee454898eb04a76bb8dc78792618640 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/5ee454898eb04a76bb8dc78792618640 2024-11-24T11:16:03,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/5ee454898eb04a76bb8dc78792618640, entries=150, sequenceid=110, filesize=11.7 K 2024-11-24T11:16:03,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d521e53077924b08b576e628fb202a2a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d521e53077924b08b576e628fb202a2a 2024-11-24T11:16:03,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d521e53077924b08b576e628fb202a2a, entries=150, sequenceid=110, filesize=11.7 K 2024-11-24T11:16:03,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f5f0024ce88b8047201380866bbadec0 in 720ms, sequenceid=110, compaction requested=false 2024-11-24T11:16:03,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:03,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-24T11:16:03,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:03,370 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:16:03,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:03,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:03,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:03,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:03,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:03,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:03,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e5e828cdfe24426583f790f6b81fd861_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446962743/Put/seqid=0 2024-11-24T11:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T11:16:03,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742182_1358 (size=12204) 2024-11-24T11:16:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:03,468 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e5e828cdfe24426583f790f6b81fd861_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e5e828cdfe24426583f790f6b81fd861_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:03,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bbb9961e37e2451d904202971a08234b, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bbb9961e37e2451d904202971a08234b is 175, key is test_row_0/A:col10/1732446962743/Put/seqid=0 2024-11-24T11:16:03,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447023454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447023459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742183_1359 (size=31005) 2024-11-24T11:16:03,525 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bbb9961e37e2451d904202971a08234b 2024-11-24T11:16:03,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/58541420803349ca82e626bd15629f89 is 50, key is test_row_0/B:col10/1732446962743/Put/seqid=0 2024-11-24T11:16:03,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742184_1360 (size=12051) 2024-11-24T11:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447023583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447023583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,595 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/58541420803349ca82e626bd15629f89 2024-11-24T11:16:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/33305dca617248529cbe346cb744b7aa is 50, key is test_row_0/C:col10/1732446962743/Put/seqid=0 2024-11-24T11:16:03,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742185_1361 (size=12051) 2024-11-24T11:16:03,670 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/33305dca617248529cbe346cb744b7aa 2024-11-24T11:16:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bbb9961e37e2451d904202971a08234b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bbb9961e37e2451d904202971a08234b 2024-11-24T11:16:03,685 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bbb9961e37e2451d904202971a08234b, entries=150, sequenceid=132, filesize=30.3 K 2024-11-24T11:16:03,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/58541420803349ca82e626bd15629f89 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/58541420803349ca82e626bd15629f89 2024-11-24T11:16:03,696 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/58541420803349ca82e626bd15629f89, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T11:16:03,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/33305dca617248529cbe346cb744b7aa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/33305dca617248529cbe346cb744b7aa 2024-11-24T11:16:03,710 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/33305dca617248529cbe346cb744b7aa, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T11:16:03,711 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f5f0024ce88b8047201380866bbadec0 in 342ms, sequenceid=132, compaction requested=true 2024-11-24T11:16:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-24T11:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-24T11:16:03,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-24T11:16:03,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-24T11:16:03,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.4310 sec 2024-11-24T11:16:03,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:16:03,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:03,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:03,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:03,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:03,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:03,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:03,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244a87fe02ed984bda89e03dbba0686241_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446963450/Put/seqid=0 2024-11-24T11:16:03,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742186_1362 (size=14794) 2024-11-24T11:16:03,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:03,867 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244a87fe02ed984bda89e03dbba0686241_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244a87fe02ed984bda89e03dbba0686241_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:03,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/8ab69c8dca5a41d880648127026c20ef, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:03,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/8ab69c8dca5a41d880648127026c20ef is 175, key is test_row_0/A:col10/1732446963450/Put/seqid=0 2024-11-24T11:16:03,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447023872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742187_1363 (size=39749) 2024-11-24T11:16:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447023873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,895 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/8ab69c8dca5a41d880648127026c20ef 2024-11-24T11:16:03,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447023876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447023887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447023890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:03,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/75e68c6769e34e1f95d5c7885eaf1f5f is 50, key is test_row_0/B:col10/1732446963450/Put/seqid=0 2024-11-24T11:16:03,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742188_1364 (size=12151) 2024-11-24T11:16:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447024004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447024004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447024005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447024005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447024015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447024224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447024224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447024224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447024224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447024233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/75e68c6769e34e1f95d5c7885eaf1f5f 2024-11-24T11:16:04,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f5c4d2f806a4464984b2e756fc8448ea is 50, key is test_row_0/C:col10/1732446963450/Put/seqid=0 2024-11-24T11:16:04,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742189_1365 (size=12151) 2024-11-24T11:16:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T11:16:04,434 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:04,434 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-24T11:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T11:16:04,436 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:04,436 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:04,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T11:16:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447024543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447024543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447024543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447024543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447024553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:04,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T11:16:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:04,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T11:16:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:04,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T11:16:04,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f5c4d2f806a4464984b2e756fc8448ea 2024-11-24T11:16:04,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/8ab69c8dca5a41d880648127026c20ef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/8ab69c8dca5a41d880648127026c20ef 2024-11-24T11:16:04,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/8ab69c8dca5a41d880648127026c20ef, entries=200, sequenceid=146, filesize=38.8 K 2024-11-24T11:16:04,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/75e68c6769e34e1f95d5c7885eaf1f5f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/75e68c6769e34e1f95d5c7885eaf1f5f 2024-11-24T11:16:04,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/75e68c6769e34e1f95d5c7885eaf1f5f, entries=150, sequenceid=146, filesize=11.9 K 2024-11-24T11:16:04,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f5c4d2f806a4464984b2e756fc8448ea as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f5c4d2f806a4464984b2e756fc8448ea 2024-11-24T11:16:04,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f5c4d2f806a4464984b2e756fc8448ea, entries=150, sequenceid=146, filesize=11.9 K 2024-11-24T11:16:04,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f5f0024ce88b8047201380866bbadec0 in 981ms, sequenceid=146, compaction requested=true 2024-11-24T11:16:04,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:04,791 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:16:04,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:04,792 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:16:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:04,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:04,793 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:16:04,793 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/A is initiating minor compaction (all files) 2024-11-24T11:16:04,793 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/A in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,793 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/4ded5e298d3045169fabbfffaae11f20, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f720290d2a1549cd9e06d754a6eae53e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bbb9961e37e2451d904202971a08234b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/8ab69c8dca5a41d880648127026c20ef] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=154.9 K 2024-11-24T11:16:04,793 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,793 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/4ded5e298d3045169fabbfffaae11f20, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f720290d2a1549cd9e06d754a6eae53e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bbb9961e37e2451d904202971a08234b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/8ab69c8dca5a41d880648127026c20ef] 2024-11-24T11:16:04,793 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:16:04,793 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/B is initiating minor compaction (all files) 2024-11-24T11:16:04,794 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/B in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,794 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/e6b4668c9d4c4587b9352607dfccca08, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/5ee454898eb04a76bb8dc78792618640, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/58541420803349ca82e626bd15629f89, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/75e68c6769e34e1f95d5c7885eaf1f5f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=47.2 K 2024-11-24T11:16:04,794 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ded5e298d3045169fabbfffaae11f20, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732446960435 2024-11-24T11:16:04,794 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b4668c9d4c4587b9352607dfccca08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732446960435 2024-11-24T11:16:04,794 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f720290d2a1549cd9e06d754a6eae53e, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732446961626 2024-11-24T11:16:04,794 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee454898eb04a76bb8dc78792618640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732446961626 2024-11-24T11:16:04,794 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb9961e37e2451d904202971a08234b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732446962632 2024-11-24T11:16:04,795 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 58541420803349ca82e626bd15629f89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732446962632 2024-11-24T11:16:04,795 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e68c6769e34e1f95d5c7885eaf1f5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732446963450 2024-11-24T11:16:04,795 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab69c8dca5a41d880648127026c20ef, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732446963423 2024-11-24T11:16:04,805 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:04,805 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:04,805 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/de0bb4db866e4694a78519e8d26dde5b is 50, key is test_row_0/B:col10/1732446963450/Put/seqid=0 2024-11-24T11:16:04,807 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124f1a63ab303884047a4aa39ccc7cd571f_f5f0024ce88b8047201380866bbadec0 store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:04,809 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124f1a63ab303884047a4aa39ccc7cd571f_f5f0024ce88b8047201380866bbadec0, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:04,809 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f1a63ab303884047a4aa39ccc7cd571f_f5f0024ce88b8047201380866bbadec0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:04,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742190_1366 (size=12425) 2024-11-24T11:16:04,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742191_1367 (size=4469) 2024-11-24T11:16:04,817 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/de0bb4db866e4694a78519e8d26dde5b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/de0bb4db866e4694a78519e8d26dde5b 2024-11-24T11:16:04,820 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5f0024ce88b8047201380866bbadec0/B of f5f0024ce88b8047201380866bbadec0 into de0bb4db866e4694a78519e8d26dde5b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:04,820 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:04,820 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/B, priority=12, startTime=1732446964792; duration=0sec 2024-11-24T11:16:04,820 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:04,820 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:B 2024-11-24T11:16:04,821 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:16:04,821 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:16:04,821 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/C is initiating minor compaction (all files) 2024-11-24T11:16:04,821 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/C in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,822 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/14c21458d18a4e9a817923f7d06cfccf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d521e53077924b08b576e628fb202a2a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/33305dca617248529cbe346cb744b7aa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f5c4d2f806a4464984b2e756fc8448ea] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=47.2 K 2024-11-24T11:16:04,822 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c21458d18a4e9a817923f7d06cfccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732446960435 2024-11-24T11:16:04,822 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d521e53077924b08b576e628fb202a2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732446961626 2024-11-24T11:16:04,822 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 33305dca617248529cbe346cb744b7aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732446962632 2024-11-24T11:16:04,822 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c4d2f806a4464984b2e756fc8448ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732446963450 2024-11-24T11:16:04,829 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:04,830 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/3128a50661d3440fbabca741dd1d552d is 50, key is test_row_0/C:col10/1732446963450/Put/seqid=0 2024-11-24T11:16:04,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742192_1368 (size=12425) 2024-11-24T11:16:04,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T11:16:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:04,894 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:16:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:04,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fca4e5ede7e648d2bbd63a9c16b42bf2_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446963888/Put/seqid=0 2024-11-24T11:16:04,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742193_1369 (size=12304) 2024-11-24T11:16:04,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:04,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fca4e5ede7e648d2bbd63a9c16b42bf2_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fca4e5ede7e648d2bbd63a9c16b42bf2_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:04,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/851a60135ac8417992110082ecad9a5b, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:04,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/851a60135ac8417992110082ecad9a5b is 175, key is test_row_0/A:col10/1732446963888/Put/seqid=0 2024-11-24T11:16:04,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742194_1370 (size=31105) 2024-11-24T11:16:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T11:16:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:05,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:05,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447025085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447025086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447025086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447025087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447025089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447025194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447025203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447025203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447025203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,214 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:05,215 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/b3e4a34452984a4d94e0b5b7454bb499 is 175, key is test_row_0/A:col10/1732446963450/Put/seqid=0 2024-11-24T11:16:05,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742195_1371 (size=31379) 2024-11-24T11:16:05,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447025213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,236 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/3128a50661d3440fbabca741dd1d552d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/3128a50661d3440fbabca741dd1d552d 2024-11-24T11:16:05,239 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5f0024ce88b8047201380866bbadec0/C of f5f0024ce88b8047201380866bbadec0 into 3128a50661d3440fbabca741dd1d552d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:05,239 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:05,239 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/C, priority=12, startTime=1732446964792; duration=0sec 2024-11-24T11:16:05,239 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:05,239 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:C 2024-11-24T11:16:05,317 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/851a60135ac8417992110082ecad9a5b 2024-11-24T11:16:05,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/d9bdc2360d4d4c1cbb38a98f18706945 is 50, key is test_row_0/B:col10/1732446963888/Put/seqid=0 2024-11-24T11:16:05,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742196_1372 (size=12151) 2024-11-24T11:16:05,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447025413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447025413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447025413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447025413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447025434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T11:16:05,622 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/b3e4a34452984a4d94e0b5b7454bb499 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b3e4a34452984a4d94e0b5b7454bb499 2024-11-24T11:16:05,627 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5f0024ce88b8047201380866bbadec0/A of f5f0024ce88b8047201380866bbadec0 into b3e4a34452984a4d94e0b5b7454bb499(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:05,627 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:05,627 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/A, priority=12, startTime=1732446964791; duration=0sec 2024-11-24T11:16:05,627 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:05,627 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:A 2024-11-24T11:16:05,725 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/d9bdc2360d4d4c1cbb38a98f18706945 2024-11-24T11:16:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447025723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447025723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447025723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447025723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/8fd18fcb971149899169c802d985993f is 50, key is test_row_0/C:col10/1732446963888/Put/seqid=0 2024-11-24T11:16:05,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742197_1373 (size=12151) 2024-11-24T11:16:05,748 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T11:16:05,749 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T11:16:05,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447025754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,135 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/8fd18fcb971149899169c802d985993f 2024-11-24T11:16:06,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/851a60135ac8417992110082ecad9a5b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/851a60135ac8417992110082ecad9a5b 2024-11-24T11:16:06,141 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/851a60135ac8417992110082ecad9a5b, entries=150, sequenceid=165, filesize=30.4 K 2024-11-24T11:16:06,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/d9bdc2360d4d4c1cbb38a98f18706945 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/d9bdc2360d4d4c1cbb38a98f18706945 2024-11-24T11:16:06,145 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/d9bdc2360d4d4c1cbb38a98f18706945, entries=150, sequenceid=165, filesize=11.9 K 2024-11-24T11:16:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/8fd18fcb971149899169c802d985993f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8fd18fcb971149899169c802d985993f 2024-11-24T11:16:06,148 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8fd18fcb971149899169c802d985993f, entries=150, sequenceid=165, filesize=11.9 K 2024-11-24T11:16:06,149 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f5f0024ce88b8047201380866bbadec0 in 1255ms, sequenceid=165, compaction requested=false 2024-11-24T11:16:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-24T11:16:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-24T11:16:06,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-24T11:16:06,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-24T11:16:06,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.7180 sec 2024-11-24T11:16:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:06,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:16:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:06,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248be806d450c546449f0d38749f5071b9_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446966236/Put/seqid=0 2024-11-24T11:16:06,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742198_1374 (size=14794) 2024-11-24T11:16:06,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447026305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447026307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447026307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447026308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447026308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447026413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447026424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447026424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447026424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447026433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T11:16:06,584 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:06,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-24T11:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T11:16:06,585 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:06,586 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:06,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:06,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447026624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447026633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447026633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447026633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,650 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:06,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248be806d450c546449f0d38749f5071b9_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248be806d450c546449f0d38749f5071b9_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:06,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/466d837b853a426bbdfd2eacdbe876c9, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:06,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/466d837b853a426bbdfd2eacdbe876c9 is 175, key is test_row_0/A:col10/1732446966236/Put/seqid=0 2024-11-24T11:16:06,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742199_1375 (size=39749) 2024-11-24T11:16:06,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447026643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T11:16:06,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:06,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:06,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:06,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:06,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T11:16:06,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447026934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447026954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447026954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447026954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447026963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:07,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:07,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/466d837b853a426bbdfd2eacdbe876c9 2024-11-24T11:16:07,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/3c600c953bd14435b68f9eaeb565719d is 50, key is test_row_0/B:col10/1732446966236/Put/seqid=0 2024-11-24T11:16:07,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742200_1376 (size=12151) 2024-11-24T11:16:07,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T11:16:07,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:07,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447027453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:07,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/3c600c953bd14435b68f9eaeb565719d 2024-11-24T11:16:07,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/93ec61873fa147c7a625758db61a8134 is 50, key is test_row_0/C:col10/1732446966236/Put/seqid=0 2024-11-24T11:16:07,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742201_1377 (size=12151) 2024-11-24T11:16:07,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447027474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:07,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447027474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447027474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447027474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:07,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T11:16:07,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:07,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/93ec61873fa147c7a625758db61a8134 2024-11-24T11:16:07,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/466d837b853a426bbdfd2eacdbe876c9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/466d837b853a426bbdfd2eacdbe876c9 2024-11-24T11:16:07,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/466d837b853a426bbdfd2eacdbe876c9, entries=200, sequenceid=182, filesize=38.8 K 2024-11-24T11:16:07,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/3c600c953bd14435b68f9eaeb565719d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/3c600c953bd14435b68f9eaeb565719d 2024-11-24T11:16:07,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/3c600c953bd14435b68f9eaeb565719d, entries=150, sequenceid=182, filesize=11.9 K 2024-11-24T11:16:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/93ec61873fa147c7a625758db61a8134 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/93ec61873fa147c7a625758db61a8134 2024-11-24T11:16:07,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/93ec61873fa147c7a625758db61a8134, entries=150, sequenceid=182, filesize=11.9 K 2024-11-24T11:16:07,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for f5f0024ce88b8047201380866bbadec0 in 1657ms, sequenceid=182, compaction requested=true 2024-11-24T11:16:07,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:07,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:07,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:07,894 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:07,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:07,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:07,894 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:07,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:07,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:07,895 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:07,895 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/A is initiating minor compaction (all files) 2024-11-24T11:16:07,895 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/A in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,896 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b3e4a34452984a4d94e0b5b7454bb499, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/851a60135ac8417992110082ecad9a5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/466d837b853a426bbdfd2eacdbe876c9] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=99.8 K 2024-11-24T11:16:07,896 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,896 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b3e4a34452984a4d94e0b5b7454bb499, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/851a60135ac8417992110082ecad9a5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/466d837b853a426bbdfd2eacdbe876c9] 2024-11-24T11:16:07,896 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:07,896 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/B is initiating minor compaction (all files) 2024-11-24T11:16:07,896 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/B in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,896 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/de0bb4db866e4694a78519e8d26dde5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/d9bdc2360d4d4c1cbb38a98f18706945, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/3c600c953bd14435b68f9eaeb565719d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=35.9 K 2024-11-24T11:16:07,897 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3e4a34452984a4d94e0b5b7454bb499, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732446963450 2024-11-24T11:16:07,897 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting de0bb4db866e4694a78519e8d26dde5b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732446963450 2024-11-24T11:16:07,897 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851a60135ac8417992110082ecad9a5b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732446963872 2024-11-24T11:16:07,897 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d9bdc2360d4d4c1cbb38a98f18706945, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732446963872 2024-11-24T11:16:07,897 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466d837b853a426bbdfd2eacdbe876c9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732446965084 2024-11-24T11:16:07,897 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c600c953bd14435b68f9eaeb565719d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732446965084 2024-11-24T11:16:07,905 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:07,905 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/26774e0cb2444ca5ba56accfd2403802 is 50, key is test_row_0/B:col10/1732446966236/Put/seqid=0 2024-11-24T11:16:07,909 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:07,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742202_1378 (size=12527) 2024-11-24T11:16:07,917 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124aaec9b3d91bc4fedaa107d2f6b5919c6_f5f0024ce88b8047201380866bbadec0 store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:07,919 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124aaec9b3d91bc4fedaa107d2f6b5919c6_f5f0024ce88b8047201380866bbadec0, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:07,919 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124aaec9b3d91bc4fedaa107d2f6b5919c6_f5f0024ce88b8047201380866bbadec0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:07,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742203_1379 (size=4469) 2024-11-24T11:16:07,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T11:16:07,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:07,960 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:16:07,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:07,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:07,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242ec62d7bb27d44548dcdb755a74a5adc_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446966304/Put/seqid=0 2024-11-24T11:16:07,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742204_1380 (size=12304) 2024-11-24T11:16:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:07,974 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242ec62d7bb27d44548dcdb755a74a5adc_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242ec62d7bb27d44548dcdb755a74a5adc_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:07,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f30ff97d206f4ba88b526b877155ac29, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:07,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f30ff97d206f4ba88b526b877155ac29 is 175, key is test_row_0/A:col10/1732446966304/Put/seqid=0 2024-11-24T11:16:07,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742205_1381 (size=31105) 2024-11-24T11:16:08,326 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/26774e0cb2444ca5ba56accfd2403802 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/26774e0cb2444ca5ba56accfd2403802 2024-11-24T11:16:08,326 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#A#compaction#319 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:08,327 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/47d269d6d60e458ba8de290060320d3e is 175, key is test_row_0/A:col10/1732446966236/Put/seqid=0 2024-11-24T11:16:08,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742206_1382 (size=31481) 2024-11-24T11:16:08,335 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/B of f5f0024ce88b8047201380866bbadec0 into 26774e0cb2444ca5ba56accfd2403802(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:08,335 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:08,335 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/B, priority=13, startTime=1732446967894; duration=0sec 2024-11-24T11:16:08,336 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:08,336 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:B 2024-11-24T11:16:08,336 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:08,336 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:08,336 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/C is initiating minor compaction (all files) 2024-11-24T11:16:08,336 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/C in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:08,337 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/3128a50661d3440fbabca741dd1d552d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8fd18fcb971149899169c802d985993f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/93ec61873fa147c7a625758db61a8134] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=35.9 K 2024-11-24T11:16:08,337 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3128a50661d3440fbabca741dd1d552d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732446963450 2024-11-24T11:16:08,338 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd18fcb971149899169c802d985993f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732446963872 2024-11-24T11:16:08,352 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ec61873fa147c7a625758db61a8134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732446965084 2024-11-24T11:16:08,365 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:08,365 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/6b32db082bc743eabda461131c212e14 is 50, key is test_row_0/C:col10/1732446966236/Put/seqid=0 2024-11-24T11:16:08,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742207_1383 (size=12527) 2024-11-24T11:16:08,378 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/6b32db082bc743eabda461131c212e14 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/6b32db082bc743eabda461131c212e14 2024-11-24T11:16:08,381 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/C of f5f0024ce88b8047201380866bbadec0 into 6b32db082bc743eabda461131c212e14(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:08,381 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:08,381 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/C, priority=13, startTime=1732446967894; duration=0sec 2024-11-24T11:16:08,381 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:08,381 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:C 2024-11-24T11:16:08,384 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f30ff97d206f4ba88b526b877155ac29 2024-11-24T11:16:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4c7b250d74d1433d88a2f75f9256770a is 50, key is test_row_0/B:col10/1732446966304/Put/seqid=0 2024-11-24T11:16:08,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742208_1384 (size=12151) 2024-11-24T11:16:08,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:08,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447028502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447028503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447028504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447028510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447028512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447028624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447028624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447028624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447028624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447028633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T11:16:08,739 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/47d269d6d60e458ba8de290060320d3e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/47d269d6d60e458ba8de290060320d3e 2024-11-24T11:16:08,743 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/A of f5f0024ce88b8047201380866bbadec0 into 47d269d6d60e458ba8de290060320d3e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:08,743 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:08,743 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/A, priority=13, startTime=1732446967894; duration=0sec 2024-11-24T11:16:08,743 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:08,743 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:A 2024-11-24T11:16:08,795 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4c7b250d74d1433d88a2f75f9256770a 2024-11-24T11:16:08,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/301c90c67ed04e64b5d24bdd15a71d5b is 50, key is test_row_0/C:col10/1732446966304/Put/seqid=0 2024-11-24T11:16:08,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742209_1385 (size=12151) 2024-11-24T11:16:08,812 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/301c90c67ed04e64b5d24bdd15a71d5b 2024-11-24T11:16:08,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f30ff97d206f4ba88b526b877155ac29 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f30ff97d206f4ba88b526b877155ac29 2024-11-24T11:16:08,818 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f30ff97d206f4ba88b526b877155ac29, entries=150, sequenceid=200, filesize=30.4 K 2024-11-24T11:16:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4c7b250d74d1433d88a2f75f9256770a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4c7b250d74d1433d88a2f75f9256770a 2024-11-24T11:16:08,825 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4c7b250d74d1433d88a2f75f9256770a, entries=150, sequenceid=200, filesize=11.9 K 2024-11-24T11:16:08,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/301c90c67ed04e64b5d24bdd15a71d5b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/301c90c67ed04e64b5d24bdd15a71d5b 2024-11-24T11:16:08,833 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/301c90c67ed04e64b5d24bdd15a71d5b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-24T11:16:08,834 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f5f0024ce88b8047201380866bbadec0 in 874ms, sequenceid=200, compaction requested=false 2024-11-24T11:16:08,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:08,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:08,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-24T11:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-24T11:16:08,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-24T11:16:08,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2500 sec 2024-11-24T11:16:08,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.2540 sec 2024-11-24T11:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:08,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:16:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:08,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246a7ee36f1fda49cc9a5b401de0882f18_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446968500/Put/seqid=0 2024-11-24T11:16:08,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742210_1386 (size=14794) 2024-11-24T11:16:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447028872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447028874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447028880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447028885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447028890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447029004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447029004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447029213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447029213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447029213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447029224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447029224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:09,265 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246a7ee36f1fda49cc9a5b401de0882f18_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a7ee36f1fda49cc9a5b401de0882f18_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:09,265 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/5abbdf2b6d42463fa78b493faeb2204a, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:09,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/5abbdf2b6d42463fa78b493faeb2204a is 175, key is test_row_0/A:col10/1732446968500/Put/seqid=0 2024-11-24T11:16:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742211_1387 (size=39749) 2024-11-24T11:16:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447029533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447029533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447029533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447029543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447029543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:09,670 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/5abbdf2b6d42463fa78b493faeb2204a 2024-11-24T11:16:09,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/8f09eea37f484ebe9f8a4159cbf951ff is 50, key is test_row_0/B:col10/1732446968500/Put/seqid=0 2024-11-24T11:16:09,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742212_1388 (size=12151) 2024-11-24T11:16:09,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/8f09eea37f484ebe9f8a4159cbf951ff 2024-11-24T11:16:09,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/78e57cb7f67843debb11a01acddd8e71 is 50, key is test_row_0/C:col10/1732446968500/Put/seqid=0 2024-11-24T11:16:09,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742213_1389 (size=12151) 2024-11-24T11:16:09,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/78e57cb7f67843debb11a01acddd8e71 2024-11-24T11:16:09,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/5abbdf2b6d42463fa78b493faeb2204a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/5abbdf2b6d42463fa78b493faeb2204a 2024-11-24T11:16:09,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/5abbdf2b6d42463fa78b493faeb2204a, entries=200, sequenceid=218, filesize=38.8 K 2024-11-24T11:16:09,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/8f09eea37f484ebe9f8a4159cbf951ff as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/8f09eea37f484ebe9f8a4159cbf951ff 2024-11-24T11:16:09,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/8f09eea37f484ebe9f8a4159cbf951ff, entries=150, sequenceid=218, filesize=11.9 K 2024-11-24T11:16:09,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/78e57cb7f67843debb11a01acddd8e71 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/78e57cb7f67843debb11a01acddd8e71 2024-11-24T11:16:09,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/78e57cb7f67843debb11a01acddd8e71, entries=150, sequenceid=218, filesize=11.9 K 2024-11-24T11:16:09,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f5f0024ce88b8047201380866bbadec0 in 878ms, sequenceid=218, compaction requested=true 2024-11-24T11:16:09,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:09,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:09,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:09,727 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:09,727 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:09,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:09,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:09,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:09,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:09,728 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:09,728 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:09,728 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/B is initiating minor compaction (all files) 2024-11-24T11:16:09,728 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/A is initiating minor compaction (all files) 2024-11-24T11:16:09,728 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/A in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:09,728 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/B in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:09,729 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/26774e0cb2444ca5ba56accfd2403802, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4c7b250d74d1433d88a2f75f9256770a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/8f09eea37f484ebe9f8a4159cbf951ff] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.0 K 2024-11-24T11:16:09,729 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/47d269d6d60e458ba8de290060320d3e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f30ff97d206f4ba88b526b877155ac29, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/5abbdf2b6d42463fa78b493faeb2204a] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=99.9 K 2024-11-24T11:16:09,729 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:09,729 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/47d269d6d60e458ba8de290060320d3e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f30ff97d206f4ba88b526b877155ac29, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/5abbdf2b6d42463fa78b493faeb2204a] 2024-11-24T11:16:09,729 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 26774e0cb2444ca5ba56accfd2403802, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732446965084 2024-11-24T11:16:09,729 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d269d6d60e458ba8de290060320d3e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732446965084 2024-11-24T11:16:09,730 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7b250d74d1433d88a2f75f9256770a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732446966304 2024-11-24T11:16:09,730 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f30ff97d206f4ba88b526b877155ac29, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732446966304 2024-11-24T11:16:09,732 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f09eea37f484ebe9f8a4159cbf951ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732446968500 2024-11-24T11:16:09,732 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5abbdf2b6d42463fa78b493faeb2204a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732446968500 2024-11-24T11:16:09,738 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:09,739 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:09,739 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/dec25fb92e55453ba575000133e83a63 is 50, key is test_row_0/B:col10/1732446968500/Put/seqid=0 2024-11-24T11:16:09,743 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124063f26d54f3940c4ac49d195ad9fc04f_f5f0024ce88b8047201380866bbadec0 store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:09,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742214_1390 (size=12629) 2024-11-24T11:16:09,745 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124063f26d54f3940c4ac49d195ad9fc04f_f5f0024ce88b8047201380866bbadec0, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:09,745 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124063f26d54f3940c4ac49d195ad9fc04f_f5f0024ce88b8047201380866bbadec0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:09,749 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/dec25fb92e55453ba575000133e83a63 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/dec25fb92e55453ba575000133e83a63 2024-11-24T11:16:09,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742215_1391 (size=4469) 2024-11-24T11:16:09,756 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/B of f5f0024ce88b8047201380866bbadec0 into dec25fb92e55453ba575000133e83a63(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:09,756 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:09,756 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/B, priority=13, startTime=1732446969727; duration=0sec 2024-11-24T11:16:09,756 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:09,756 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:B 2024-11-24T11:16:09,756 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:09,757 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:09,757 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/C is initiating minor compaction (all files) 2024-11-24T11:16:09,757 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/C in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:09,757 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/6b32db082bc743eabda461131c212e14, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/301c90c67ed04e64b5d24bdd15a71d5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/78e57cb7f67843debb11a01acddd8e71] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.0 K 2024-11-24T11:16:09,757 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b32db082bc743eabda461131c212e14, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732446965084 2024-11-24T11:16:09,758 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 301c90c67ed04e64b5d24bdd15a71d5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732446966304 2024-11-24T11:16:09,760 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e57cb7f67843debb11a01acddd8e71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732446968500 2024-11-24T11:16:09,766 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:09,766 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/a013709057af44539808d9d98d91a81d is 50, key is test_row_0/C:col10/1732446968500/Put/seqid=0 2024-11-24T11:16:09,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742216_1392 (size=12629) 2024-11-24T11:16:09,790 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/a013709057af44539808d9d98d91a81d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/a013709057af44539808d9d98d91a81d 2024-11-24T11:16:09,802 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/C of f5f0024ce88b8047201380866bbadec0 into a013709057af44539808d9d98d91a81d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:09,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:09,802 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/C, priority=13, startTime=1732446969727; duration=0sec 2024-11-24T11:16:09,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:09,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:C 2024-11-24T11:16:10,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:16:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:10,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:10,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:10,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:10,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:10,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:10,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:10,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ee31212c5ddf4457a9119d679c8ccb5e_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446970047/Put/seqid=0 2024-11-24T11:16:10,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742217_1393 (size=17284) 2024-11-24T11:16:10,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:10,078 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ee31212c5ddf4457a9119d679c8ccb5e_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ee31212c5ddf4457a9119d679c8ccb5e_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:10,079 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bf86f46b478445caa4075eedc5221ec3, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:10,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bf86f46b478445caa4075eedc5221ec3 is 175, key is test_row_0/A:col10/1732446970047/Put/seqid=0 2024-11-24T11:16:10,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742218_1394 (size=48389) 2024-11-24T11:16:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447030073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447030075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447030077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447030086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447030087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,154 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:10,155 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/b7bda0738918403f8e54ada17dc58089 is 175, key is test_row_0/A:col10/1732446968500/Put/seqid=0 2024-11-24T11:16:10,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742219_1395 (size=31583) 2024-11-24T11:16:10,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447030193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447030193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447030194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447030203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447030204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447030414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447030415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447030415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447030415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447030424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bf86f46b478445caa4075eedc5221ec3 2024-11-24T11:16:10,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1792fe7e7bfc4728ac8d8ee5be96b8eb is 50, key is test_row_0/B:col10/1732446970047/Put/seqid=0 2024-11-24T11:16:10,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742220_1396 (size=12151) 2024-11-24T11:16:10,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1792fe7e7bfc4728ac8d8ee5be96b8eb 2024-11-24T11:16:10,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/e8fbe74785ec4a36ba911db4858140b1 is 50, key is test_row_0/C:col10/1732446970047/Put/seqid=0 2024-11-24T11:16:10,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742221_1397 (size=12151) 2024-11-24T11:16:10,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/e8fbe74785ec4a36ba911db4858140b1 2024-11-24T11:16:10,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/bf86f46b478445caa4075eedc5221ec3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bf86f46b478445caa4075eedc5221ec3 2024-11-24T11:16:10,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bf86f46b478445caa4075eedc5221ec3, entries=250, sequenceid=239, filesize=47.3 K 2024-11-24T11:16:10,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1792fe7e7bfc4728ac8d8ee5be96b8eb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1792fe7e7bfc4728ac8d8ee5be96b8eb 2024-11-24T11:16:10,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1792fe7e7bfc4728ac8d8ee5be96b8eb, entries=150, sequenceid=239, filesize=11.9 K 2024-11-24T11:16:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/e8fbe74785ec4a36ba911db4858140b1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/e8fbe74785ec4a36ba911db4858140b1 2024-11-24T11:16:10,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/e8fbe74785ec4a36ba911db4858140b1, entries=150, sequenceid=239, filesize=11.9 K 2024-11-24T11:16:10,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f5f0024ce88b8047201380866bbadec0 in 506ms, sequenceid=239, compaction requested=false 2024-11-24T11:16:10,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:10,602 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/b7bda0738918403f8e54ada17dc58089 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b7bda0738918403f8e54ada17dc58089 2024-11-24T11:16:10,606 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/A of f5f0024ce88b8047201380866bbadec0 into b7bda0738918403f8e54ada17dc58089(size=30.8 K), total size for store is 78.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:10,606 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:10,606 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/A, priority=13, startTime=1732446969727; duration=0sec 2024-11-24T11:16:10,607 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:10,607 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:A 2024-11-24T11:16:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T11:16:10,744 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:10,745 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-24T11:16:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T11:16:10,746 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:10,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:16:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:10,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:10,751 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:10,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:10,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240cc8e55bdd484f75869dfba9c3bce231_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446970079/Put/seqid=0 2024-11-24T11:16:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447030792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742222_1398 (size=14794) 2024-11-24T11:16:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447030794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447030796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:10,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447030799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447030804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240cc8e55bdd484f75869dfba9c3bce231_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240cc8e55bdd484f75869dfba9c3bce231_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:10,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/e9d7eb0c573947f1b337b0bab006e579, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:10,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/e9d7eb0c573947f1b337b0bab006e579 is 175, key is test_row_0/A:col10/1732446970079/Put/seqid=0 2024-11-24T11:16:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T11:16:10,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742223_1399 (size=39749) 2024-11-24T11:16:10,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T11:16:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:10,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:10,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447030914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447030924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447030924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447030924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447030924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T11:16:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T11:16:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447031144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447031144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447031144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447031144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447031144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T11:16:11,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:11,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:11,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:11,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/e9d7eb0c573947f1b337b0bab006e579 2024-11-24T11:16:11,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/807678bc80f6448a87c1edaf1a7a65d9 is 50, key is test_row_0/B:col10/1732446970079/Put/seqid=0 2024-11-24T11:16:11,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742224_1400 (size=12151) 2024-11-24T11:16:11,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/807678bc80f6448a87c1edaf1a7a65d9 2024-11-24T11:16:11,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f9e3db7dab104c389918dbe30eb54fd2 is 50, key is test_row_0/C:col10/1732446970079/Put/seqid=0 2024-11-24T11:16:11,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742225_1401 (size=12151) 2024-11-24T11:16:11,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f9e3db7dab104c389918dbe30eb54fd2 2024-11-24T11:16:11,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/e9d7eb0c573947f1b337b0bab006e579 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/e9d7eb0c573947f1b337b0bab006e579 2024-11-24T11:16:11,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/e9d7eb0c573947f1b337b0bab006e579, entries=200, sequenceid=254, filesize=38.8 K 2024-11-24T11:16:11,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/807678bc80f6448a87c1edaf1a7a65d9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/807678bc80f6448a87c1edaf1a7a65d9 2024-11-24T11:16:11,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/807678bc80f6448a87c1edaf1a7a65d9, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T11:16:11,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f9e3db7dab104c389918dbe30eb54fd2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f9e3db7dab104c389918dbe30eb54fd2 2024-11-24T11:16:11,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f9e3db7dab104c389918dbe30eb54fd2, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T11:16:11,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f5f0024ce88b8047201380866bbadec0 in 606ms, sequenceid=254, compaction requested=true 2024-11-24T11:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:11,354 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:11,354 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:11,356 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:11,356 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/A is initiating minor compaction (all files) 2024-11-24T11:16:11,356 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/A in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,357 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b7bda0738918403f8e54ada17dc58089, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bf86f46b478445caa4075eedc5221ec3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/e9d7eb0c573947f1b337b0bab006e579] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=116.9 K 2024-11-24T11:16:11,357 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,357 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b7bda0738918403f8e54ada17dc58089, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bf86f46b478445caa4075eedc5221ec3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/e9d7eb0c573947f1b337b0bab006e579] 2024-11-24T11:16:11,357 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:11,357 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/B is initiating minor compaction (all files) 2024-11-24T11:16:11,357 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/B in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,358 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/dec25fb92e55453ba575000133e83a63, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1792fe7e7bfc4728ac8d8ee5be96b8eb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/807678bc80f6448a87c1edaf1a7a65d9] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.1 K 2024-11-24T11:16:11,358 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7bda0738918403f8e54ada17dc58089, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732446968500 2024-11-24T11:16:11,358 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting dec25fb92e55453ba575000133e83a63, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732446968500 2024-11-24T11:16:11,358 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf86f46b478445caa4075eedc5221ec3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732446968879 2024-11-24T11:16:11,358 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1792fe7e7bfc4728ac8d8ee5be96b8eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732446968879 2024-11-24T11:16:11,358 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9d7eb0c573947f1b337b0bab006e579, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446970067 2024-11-24T11:16:11,358 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 807678bc80f6448a87c1edaf1a7a65d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446970079 2024-11-24T11:16:11,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T11:16:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,371 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:16:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T11:16:11,378 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:11,379 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1a53abef6eec4fd2aa117f5e3fcda48a is 50, key is test_row_0/B:col10/1732446970079/Put/seqid=0 2024-11-24T11:16:11,386 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:11,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742226_1402 (size=12731) 2024-11-24T11:16:11,397 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112496c6c817695245edbf8a1817d27f2fe1_f5f0024ce88b8047201380866bbadec0 store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:11,399 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112496c6c817695245edbf8a1817d27f2fe1_f5f0024ce88b8047201380866bbadec0, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:11,399 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112496c6c817695245edbf8a1817d27f2fe1_f5f0024ce88b8047201380866bbadec0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ef94232e708b4bb09728ed6f49fff5d5_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446970788/Put/seqid=0 2024-11-24T11:16:11,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742227_1403 (size=4469) 2024-11-24T11:16:11,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742228_1404 (size=12454) 2024-11-24T11:16:11,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:11,441 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ef94232e708b4bb09728ed6f49fff5d5_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ef94232e708b4bb09728ed6f49fff5d5_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:11,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/53aa94687eda4d06a91fa96723597b0b, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:11,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/53aa94687eda4d06a91fa96723597b0b is 175, key is test_row_0/A:col10/1732446970788/Put/seqid=0 2024-11-24T11:16:11,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742229_1405 (size=31255) 2024-11-24T11:16:11,450 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/53aa94687eda4d06a91fa96723597b0b 2024-11-24T11:16:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4ab176f0decf441d8e1b9b50a3d9d79a is 50, key is test_row_0/B:col10/1732446970788/Put/seqid=0 2024-11-24T11:16:11,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742230_1406 (size=12301) 2024-11-24T11:16:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:11,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447031484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447031485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447031490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447031491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447031492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447031604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447031606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447031606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447031614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447031614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,797 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/1a53abef6eec4fd2aa117f5e3fcda48a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1a53abef6eec4fd2aa117f5e3fcda48a 2024-11-24T11:16:11,801 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/B of f5f0024ce88b8047201380866bbadec0 into 1a53abef6eec4fd2aa117f5e3fcda48a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:11,801 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:11,801 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/B, priority=13, startTime=1732446971354; duration=0sec 2024-11-24T11:16:11,801 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:11,801 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:B 2024-11-24T11:16:11,801 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:11,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:11,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/C is initiating minor compaction (all files) 2024-11-24T11:16:11,802 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/C in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,802 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/a013709057af44539808d9d98d91a81d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/e8fbe74785ec4a36ba911db4858140b1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f9e3db7dab104c389918dbe30eb54fd2] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.1 K 2024-11-24T11:16:11,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a013709057af44539808d9d98d91a81d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732446968500 2024-11-24T11:16:11,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fbe74785ec4a36ba911db4858140b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732446968879 2024-11-24T11:16:11,803 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e3db7dab104c389918dbe30eb54fd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446970079 2024-11-24T11:16:11,809 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#C#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:11,809 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/786b1bcb27354913b352c19e485c16b6 is 50, key is test_row_0/C:col10/1732446970079/Put/seqid=0 2024-11-24T11:16:11,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447031813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742231_1407 (size=12731) 2024-11-24T11:16:11,829 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:11,830 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/44c030f57faf4130a749e278f987143a is 175, key is test_row_0/A:col10/1732446970079/Put/seqid=0 2024-11-24T11:16:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447031823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447031824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,841 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/786b1bcb27354913b352c19e485c16b6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/786b1bcb27354913b352c19e485c16b6 2024-11-24T11:16:11,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447031834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,845 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/C of f5f0024ce88b8047201380866bbadec0 into 786b1bcb27354913b352c19e485c16b6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:11,845 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:11,845 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/C, priority=13, startTime=1732446971354; duration=0sec 2024-11-24T11:16:11,845 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:11,845 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:C 2024-11-24T11:16:11,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447031834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:11,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742232_1408 (size=31685) 2024-11-24T11:16:11,862 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4ab176f0decf441d8e1b9b50a3d9d79a 2024-11-24T11:16:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f1acdb2e560b481aaa5406cceac01b59 is 50, key is test_row_0/C:col10/1732446970788/Put/seqid=0 2024-11-24T11:16:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742233_1409 (size=12301) 2024-11-24T11:16:11,875 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f1acdb2e560b481aaa5406cceac01b59 2024-11-24T11:16:11,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/53aa94687eda4d06a91fa96723597b0b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/53aa94687eda4d06a91fa96723597b0b 2024-11-24T11:16:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T11:16:11,885 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/53aa94687eda4d06a91fa96723597b0b, entries=150, sequenceid=272, filesize=30.5 K 2024-11-24T11:16:11,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/4ab176f0decf441d8e1b9b50a3d9d79a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4ab176f0decf441d8e1b9b50a3d9d79a 2024-11-24T11:16:11,890 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4ab176f0decf441d8e1b9b50a3d9d79a, entries=150, sequenceid=272, filesize=12.0 K 2024-11-24T11:16:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/f1acdb2e560b481aaa5406cceac01b59 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f1acdb2e560b481aaa5406cceac01b59 2024-11-24T11:16:11,894 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f1acdb2e560b481aaa5406cceac01b59, entries=150, sequenceid=272, filesize=12.0 K 2024-11-24T11:16:11,895 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f5f0024ce88b8047201380866bbadec0 in 524ms, sequenceid=272, compaction requested=false 2024-11-24T11:16:11,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:11,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:11,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-24T11:16:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-24T11:16:11,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-24T11:16:11,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1450 sec 2024-11-24T11:16:11,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.1540 sec 2024-11-24T11:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:12,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:16:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:12,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:12,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:12,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:12,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:12,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241b912693c4c14f419abb581d6a9966d1_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446971490/Put/seqid=0 2024-11-24T11:16:12,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742234_1410 (size=14994) 2024-11-24T11:16:12,161 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:12,164 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241b912693c4c14f419abb581d6a9966d1_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241b912693c4c14f419abb581d6a9966d1_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:12,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/489837606f7c41e7acfab4806a1c9236, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:12,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/489837606f7c41e7acfab4806a1c9236 is 175, key is test_row_0/A:col10/1732446971490/Put/seqid=0 2024-11-24T11:16:12,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742235_1411 (size=39949) 2024-11-24T11:16:12,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447032172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447032173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447032173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447032176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447032178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,264 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/44c030f57faf4130a749e278f987143a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/44c030f57faf4130a749e278f987143a 2024-11-24T11:16:12,269 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/A of f5f0024ce88b8047201380866bbadec0 into 44c030f57faf4130a749e278f987143a(size=30.9 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:12,269 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:12,269 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/A, priority=13, startTime=1732446971354; duration=0sec 2024-11-24T11:16:12,269 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:12,269 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:A 2024-11-24T11:16:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447032294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447032295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447032295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447032295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447032295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447032513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447032513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447032513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447032513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447032513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,577 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/489837606f7c41e7acfab4806a1c9236 2024-11-24T11:16:12,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/0481016494b543c292b918b982d02e9b is 50, key is test_row_0/B:col10/1732446971490/Put/seqid=0 2024-11-24T11:16:12,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742236_1412 (size=12301) 2024-11-24T11:16:12,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447032834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447032834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447032834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447032834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447032834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T11:16:12,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:12,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-24T11:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T11:16:12,895 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:12,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:12,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:12,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/0481016494b543c292b918b982d02e9b 2024-11-24T11:16:12,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/8a0322ce44c84512b998cb7745f6c8d4 is 50, key is test_row_0/C:col10/1732446971490/Put/seqid=0 2024-11-24T11:16:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T11:16:13,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742237_1413 (size=12301) 2024-11-24T11:16:13,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/8a0322ce44c84512b998cb7745f6c8d4 2024-11-24T11:16:13,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/489837606f7c41e7acfab4806a1c9236 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/489837606f7c41e7acfab4806a1c9236 2024-11-24T11:16:13,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/489837606f7c41e7acfab4806a1c9236, entries=200, sequenceid=289, filesize=39.0 K 2024-11-24T11:16:13,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/0481016494b543c292b918b982d02e9b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0481016494b543c292b918b982d02e9b 2024-11-24T11:16:13,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0481016494b543c292b918b982d02e9b, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T11:16:13,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/8a0322ce44c84512b998cb7745f6c8d4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8a0322ce44c84512b998cb7745f6c8d4 2024-11-24T11:16:13,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T11:16:13,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:13,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:13,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:13,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:13,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:13,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8a0322ce44c84512b998cb7745f6c8d4, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T11:16:13,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f5f0024ce88b8047201380866bbadec0 in 919ms, sequenceid=289, compaction requested=true 2024-11-24T11:16:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:13,058 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:13,058 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:13,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:13,060 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:13,060 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:13,060 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/B is initiating minor compaction (all files) 2024-11-24T11:16:13,060 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/A is initiating minor compaction (all files) 2024-11-24T11:16:13,060 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/A in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:13,060 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/B in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:13,060 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1a53abef6eec4fd2aa117f5e3fcda48a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4ab176f0decf441d8e1b9b50a3d9d79a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0481016494b543c292b918b982d02e9b] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.5 K 2024-11-24T11:16:13,060 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/44c030f57faf4130a749e278f987143a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/53aa94687eda4d06a91fa96723597b0b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/489837606f7c41e7acfab4806a1c9236] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=100.5 K 2024-11-24T11:16:13,060 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:13,060 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/44c030f57faf4130a749e278f987143a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/53aa94687eda4d06a91fa96723597b0b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/489837606f7c41e7acfab4806a1c9236] 2024-11-24T11:16:13,062 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a53abef6eec4fd2aa117f5e3fcda48a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446970079 2024-11-24T11:16:13,062 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44c030f57faf4130a749e278f987143a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446970079 2024-11-24T11:16:13,063 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53aa94687eda4d06a91fa96723597b0b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732446970788 2024-11-24T11:16:13,063 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab176f0decf441d8e1b9b50a3d9d79a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732446970788 2024-11-24T11:16:13,063 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489837606f7c41e7acfab4806a1c9236, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446971483 2024-11-24T11:16:13,065 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 0481016494b543c292b918b982d02e9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446971490 2024-11-24T11:16:13,089 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:13,090 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:13,090 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/314015879b23416ea24eb8171790edc7 is 50, key is test_row_0/B:col10/1732446971490/Put/seqid=0 2024-11-24T11:16:13,097 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112422d7286902ad48e495173f992ec06c0a_f5f0024ce88b8047201380866bbadec0 store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:13,100 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112422d7286902ad48e495173f992ec06c0a_f5f0024ce88b8047201380866bbadec0, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:13,100 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112422d7286902ad48e495173f992ec06c0a_f5f0024ce88b8047201380866bbadec0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:13,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742238_1414 (size=12983) 2024-11-24T11:16:13,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742239_1415 (size=4469) 2024-11-24T11:16:13,145 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#A#compaction#346 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:13,146 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/439f4680f73a4572af9def55f8aa6346 is 175, key is test_row_0/A:col10/1732446971490/Put/seqid=0 2024-11-24T11:16:13,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742240_1416 (size=31937) 2024-11-24T11:16:13,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T11:16:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:16:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:13,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T11:16:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124588bad5180d147a2a45a6e0f097ac045_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446972176/Put/seqid=0 2024-11-24T11:16:13,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742241_1417 (size=12454) 2024-11-24T11:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:13,235 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124588bad5180d147a2a45a6e0f097ac045_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124588bad5180d147a2a45a6e0f097ac045_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:13,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f5b435d624424ed294b4c7b76e5884fb, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:13,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f5b435d624424ed294b4c7b76e5884fb is 175, key is test_row_0/A:col10/1732446972176/Put/seqid=0 2024-11-24T11:16:13,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742242_1418 (size=31255) 2024-11-24T11:16:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:13,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:13,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447033376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447033377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447033377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447033383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447033384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447033494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447033494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447033494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447033494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447033503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T11:16:13,547 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/314015879b23416ea24eb8171790edc7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/314015879b23416ea24eb8171790edc7 2024-11-24T11:16:13,551 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/B of f5f0024ce88b8047201380866bbadec0 into 314015879b23416ea24eb8171790edc7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:13,551 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:13,551 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/B, priority=13, startTime=1732446973058; duration=0sec 2024-11-24T11:16:13,551 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:13,551 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:B 2024-11-24T11:16:13,551 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:13,552 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:13,552 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/C is initiating minor compaction (all files) 2024-11-24T11:16:13,552 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/C in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:13,552 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/786b1bcb27354913b352c19e485c16b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f1acdb2e560b481aaa5406cceac01b59, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8a0322ce44c84512b998cb7745f6c8d4] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.5 K 2024-11-24T11:16:13,552 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 786b1bcb27354913b352c19e485c16b6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446970079 2024-11-24T11:16:13,552 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f1acdb2e560b481aaa5406cceac01b59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732446970788 2024-11-24T11:16:13,552 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0322ce44c84512b998cb7745f6c8d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446971490 2024-11-24T11:16:13,558 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#C#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:13,558 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/863e0450bee6467595dd926d32621f55 is 50, key is test_row_0/C:col10/1732446971490/Put/seqid=0 2024-11-24T11:16:13,561 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/439f4680f73a4572af9def55f8aa6346 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/439f4680f73a4572af9def55f8aa6346 2024-11-24T11:16:13,564 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/A of f5f0024ce88b8047201380866bbadec0 into 439f4680f73a4572af9def55f8aa6346(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:13,564 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:13,565 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/A, priority=13, startTime=1732446973058; duration=0sec 2024-11-24T11:16:13,565 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:13,565 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:A 2024-11-24T11:16:13,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742243_1419 (size=12983) 2024-11-24T11:16:13,642 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f5b435d624424ed294b4c7b76e5884fb 2024-11-24T11:16:13,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/6b903854a40543c5a26cf31d5351ab16 is 50, key is test_row_0/B:col10/1732446972176/Put/seqid=0 2024-11-24T11:16:13,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742244_1420 (size=12301) 2024-11-24T11:16:13,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447033713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447033713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447033713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447033714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447033714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:13,970 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/863e0450bee6467595dd926d32621f55 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/863e0450bee6467595dd926d32621f55 2024-11-24T11:16:13,974 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/C of f5f0024ce88b8047201380866bbadec0 into 863e0450bee6467595dd926d32621f55(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:13,974 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:13,974 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/C, priority=13, startTime=1732446973058; duration=0sec 2024-11-24T11:16:13,974 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:13,974 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:C 2024-11-24T11:16:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447034023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T11:16:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447034033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447034033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447034034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447034034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,056 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/6b903854a40543c5a26cf31d5351ab16 2024-11-24T11:16:14,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/9f2078c2764e4398a7705069f3c2f774 is 50, key is test_row_0/C:col10/1732446972176/Put/seqid=0 2024-11-24T11:16:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742245_1421 (size=12301) 2024-11-24T11:16:14,472 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/9f2078c2764e4398a7705069f3c2f774 2024-11-24T11:16:14,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/f5b435d624424ed294b4c7b76e5884fb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f5b435d624424ed294b4c7b76e5884fb 2024-11-24T11:16:14,478 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f5b435d624424ed294b4c7b76e5884fb, entries=150, sequenceid=307, filesize=30.5 K 2024-11-24T11:16:14,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/6b903854a40543c5a26cf31d5351ab16 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/6b903854a40543c5a26cf31d5351ab16 2024-11-24T11:16:14,482 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/6b903854a40543c5a26cf31d5351ab16, entries=150, sequenceid=307, filesize=12.0 K 2024-11-24T11:16:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/9f2078c2764e4398a7705069f3c2f774 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/9f2078c2764e4398a7705069f3c2f774 2024-11-24T11:16:14,486 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/9f2078c2764e4398a7705069f3c2f774, entries=150, sequenceid=307, filesize=12.0 K 2024-11-24T11:16:14,487 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f5f0024ce88b8047201380866bbadec0 in 1286ms, sequenceid=307, compaction requested=false 2024-11-24T11:16:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-24T11:16:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-24T11:16:14,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-24T11:16:14,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5920 sec 2024-11-24T11:16:14,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.5970 sec 2024-11-24T11:16:14,519 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T11:16:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:14,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:16:14,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:14,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112421e568c977ac4be59427917cac76d144_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446973383/Put/seqid=0 2024-11-24T11:16:14,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742246_1422 (size=14994) 2024-11-24T11:16:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447034576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447034593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447034595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447034597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447034600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447034703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447034723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447034734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447034734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447034734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447034913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447034944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447034953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447034953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447034953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:14,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:14,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112421e568c977ac4be59427917cac76d144_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112421e568c977ac4be59427917cac76d144_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:14,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/022419fcfe1f46d2b0daee965ebf8925, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:14,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/022419fcfe1f46d2b0daee965ebf8925 is 175, key is test_row_0/A:col10/1732446973383/Put/seqid=0 2024-11-24T11:16:14,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742247_1423 (size=39949) 2024-11-24T11:16:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T11:16:15,043 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:15,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-24T11:16:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T11:16:15,050 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:15,051 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:15,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T11:16:15,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447035224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447035253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447035263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447035274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447035274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T11:16:15,378 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/022419fcfe1f46d2b0daee965ebf8925 2024-11-24T11:16:15,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/833e75c5fd694b2692bdbfbe7353dc38 is 50, key is test_row_0/B:col10/1732446973383/Put/seqid=0 2024-11-24T11:16:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742248_1424 (size=12301) 2024-11-24T11:16:15,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:15,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:15,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T11:16:15,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57724 deadline: 1732447035733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57714 deadline: 1732447035773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/833e75c5fd694b2692bdbfbe7353dc38 2024-11-24T11:16:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57646 deadline: 1732447035784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57670 deadline: 1732447035784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe is 50, key is test_row_0/C:col10/1732446973383/Put/seqid=0 2024-11-24T11:16:15,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742249_1425 (size=12301) 2024-11-24T11:16:15,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57704 deadline: 1732447035793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:16:15,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:15,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:15,982 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-24T11:16:15,982 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:15,982 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,982 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:15,987 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-24T11:16:15,987 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:15,987 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,988 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:15,990 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-24T11:16:15,990 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:15,990 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,990 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:15,993 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-11-24T11:16:15,993 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:15,993 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,994 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:15,999 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-24T11:16:15,999 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:15,999 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,999 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:15,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:16,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:16,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:16,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:16,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:16,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:16,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T11:16:16,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe 2024-11-24T11:16:16,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/022419fcfe1f46d2b0daee965ebf8925 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/022419fcfe1f46d2b0daee965ebf8925 2024-11-24T11:16:16,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/022419fcfe1f46d2b0daee965ebf8925, entries=200, sequenceid=326, filesize=39.0 K 2024-11-24T11:16:16,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/833e75c5fd694b2692bdbfbe7353dc38 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/833e75c5fd694b2692bdbfbe7353dc38 2024-11-24T11:16:16,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/833e75c5fd694b2692bdbfbe7353dc38, entries=150, sequenceid=326, filesize=12.0 K 2024-11-24T11:16:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe 2024-11-24T11:16:16,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe, entries=150, sequenceid=326, filesize=12.0 K 2024-11-24T11:16:16,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for f5f0024ce88b8047201380866bbadec0 in 1664ms, sequenceid=326, compaction requested=true 2024-11-24T11:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:16,212 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:16,212 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5f0024ce88b8047201380866bbadec0:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/A is initiating minor compaction (all files) 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/B is initiating minor compaction (all files) 2024-11-24T11:16:16,213 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/B in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:16,213 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/A in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:16,213 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/439f4680f73a4572af9def55f8aa6346, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f5b435d624424ed294b4c7b76e5884fb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/022419fcfe1f46d2b0daee965ebf8925] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=100.7 K 2024-11-24T11:16:16,213 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/314015879b23416ea24eb8171790edc7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/6b903854a40543c5a26cf31d5351ab16, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/833e75c5fd694b2692bdbfbe7353dc38] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.7 K 2024-11-24T11:16:16,213 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/439f4680f73a4572af9def55f8aa6346, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f5b435d624424ed294b4c7b76e5884fb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/022419fcfe1f46d2b0daee965ebf8925] 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 314015879b23416ea24eb8171790edc7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446971490 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439f4680f73a4572af9def55f8aa6346, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446971490 2024-11-24T11:16:16,213 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b903854a40543c5a26cf31d5351ab16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732446972169 2024-11-24T11:16:16,214 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5b435d624424ed294b4c7b76e5884fb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732446972169 2024-11-24T11:16:16,214 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 833e75c5fd694b2692bdbfbe7353dc38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446973381 2024-11-24T11:16:16,214 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 022419fcfe1f46d2b0daee965ebf8925, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446973374 2024-11-24T11:16:16,221 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:16,221 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:16,222 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/2b067cbbab6742d99979b1373df436e8 is 50, key is test_row_0/B:col10/1732446973383/Put/seqid=0 2024-11-24T11:16:16,223 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411248e1c81e3706e4431af3c896563e67415_f5f0024ce88b8047201380866bbadec0 store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:16,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742250_1426 (size=13085) 2024-11-24T11:16:16,244 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411248e1c81e3706e4431af3c896563e67415_f5f0024ce88b8047201380866bbadec0, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:16,244 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248e1c81e3706e4431af3c896563e67415_f5f0024ce88b8047201380866bbadec0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:16,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742251_1427 (size=4469) 2024-11-24T11:16:16,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T11:16:16,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:16,277 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:16:16,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:16,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:16,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:16,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:16,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:16,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241ee6e53cea914bfe84d1a9966cfd1987_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446974594/Put/seqid=0 2024-11-24T11:16:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742252_1428 (size=12454) 2024-11-24T11:16:16,631 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/2b067cbbab6742d99979b1373df436e8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/2b067cbbab6742d99979b1373df436e8 2024-11-24T11:16:16,637 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/B of f5f0024ce88b8047201380866bbadec0 into 2b067cbbab6742d99979b1373df436e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:16,637 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:16,637 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/B, priority=13, startTime=1732446976212; duration=0sec 2024-11-24T11:16:16,637 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:16,637 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:B 2024-11-24T11:16:16,638 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:16,639 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:16,639 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): f5f0024ce88b8047201380866bbadec0/C is initiating minor compaction (all files) 2024-11-24T11:16:16,639 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5f0024ce88b8047201380866bbadec0/C in TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:16,639 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/863e0450bee6467595dd926d32621f55, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/9f2078c2764e4398a7705069f3c2f774, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp, totalSize=36.7 K 2024-11-24T11:16:16,640 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 863e0450bee6467595dd926d32621f55, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446971490 2024-11-24T11:16:16,640 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2078c2764e4398a7705069f3c2f774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732446972169 2024-11-24T11:16:16,640 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d5c9c6cdd9ec44a4b5732c79ed6d25fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732446973381 2024-11-24T11:16:16,647 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:16,647 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/623e18adfae6478a9c3298f5da965b30 is 50, key is test_row_0/C:col10/1732446973383/Put/seqid=0 2024-11-24T11:16:16,648 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5f0024ce88b8047201380866bbadec0#A#compaction#355 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:16,649 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/9730fc8261504a5a95360d1a1cb070a1 is 175, key is test_row_0/A:col10/1732446973383/Put/seqid=0 2024-11-24T11:16:16,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742253_1429 (size=13085) 2024-11-24T11:16:16,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742254_1430 (size=32039) 2024-11-24T11:16:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:16,692 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241ee6e53cea914bfe84d1a9966cfd1987_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241ee6e53cea914bfe84d1a9966cfd1987_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/50d9760b36a3435d81712c242d7bb437, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:16,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/50d9760b36a3435d81712c242d7bb437 is 175, key is test_row_0/A:col10/1732446974594/Put/seqid=0 2024-11-24T11:16:16,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742255_1431 (size=31255) 2024-11-24T11:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. as already flushing 2024-11-24T11:16:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:16,755 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-11-24T11:16:16,755 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:16,755 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,755 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:16,785 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-11-24T11:16:16,785 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:16,785 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,785 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:16,805 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-24T11:16:16,805 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-11-24T11:16:16,805 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:16,805 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:16,805 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,805 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,805 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,805 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:16,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:16,827 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-11-24T11:16:16,827 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:16,827 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,827 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:16,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:17,054 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/623e18adfae6478a9c3298f5da965b30 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/623e18adfae6478a9c3298f5da965b30 2024-11-24T11:16:17,055 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/9730fc8261504a5a95360d1a1cb070a1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/9730fc8261504a5a95360d1a1cb070a1 2024-11-24T11:16:17,058 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/C of f5f0024ce88b8047201380866bbadec0 into 623e18adfae6478a9c3298f5da965b30(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:17,058 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:17,058 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5f0024ce88b8047201380866bbadec0/A of f5f0024ce88b8047201380866bbadec0 into 9730fc8261504a5a95360d1a1cb070a1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:17,058 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/C, priority=13, startTime=1732446976212; duration=0sec 2024-11-24T11:16:17,058 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:17,058 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., storeName=f5f0024ce88b8047201380866bbadec0/A, priority=13, startTime=1732446976212; duration=0sec 2024-11-24T11:16:17,058 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:17,058 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:C 2024-11-24T11:16:17,058 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:17,058 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5f0024ce88b8047201380866bbadec0:A 2024-11-24T11:16:17,097 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/50d9760b36a3435d81712c242d7bb437 2024-11-24T11:16:17,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/0e34990cb0bb43b9a5aa8c939dda2610 is 50, key is test_row_0/B:col10/1732446974594/Put/seqid=0 2024-11-24T11:16:17,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742256_1432 (size=12301) 2024-11-24T11:16:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T11:16:17,508 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/0e34990cb0bb43b9a5aa8c939dda2610 2024-11-24T11:16:17,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/421048acf20b4007bf8c1077199477db is 50, key is test_row_0/C:col10/1732446974594/Put/seqid=0 2024-11-24T11:16:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742257_1433 (size=12301) 2024-11-24T11:16:17,918 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/421048acf20b4007bf8c1077199477db 2024-11-24T11:16:17,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/50d9760b36a3435d81712c242d7bb437 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/50d9760b36a3435d81712c242d7bb437 2024-11-24T11:16:17,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/50d9760b36a3435d81712c242d7bb437, entries=150, sequenceid=343, filesize=30.5 K 2024-11-24T11:16:17,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/0e34990cb0bb43b9a5aa8c939dda2610 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0e34990cb0bb43b9a5aa8c939dda2610 2024-11-24T11:16:17,929 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0e34990cb0bb43b9a5aa8c939dda2610, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T11:16:17,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/421048acf20b4007bf8c1077199477db as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/421048acf20b4007bf8c1077199477db 2024-11-24T11:16:17,933 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/421048acf20b4007bf8c1077199477db, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T11:16:17,934 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=33.54 KB/34350 for f5f0024ce88b8047201380866bbadec0 in 1657ms, sequenceid=343, compaction requested=false 2024-11-24T11:16:17,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for f5f0024ce88b8047201380866bbadec0: 2024-11-24T11:16:17,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:17,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-24T11:16:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-24T11:16:17,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-24T11:16:17,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8840 sec 2024-11-24T11:16:17,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.8930 sec 2024-11-24T11:16:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T11:16:19,214 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:19,214 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1014 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3040 rows 2024-11-24T11:16:19,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/42d82f541bcc4f868f47e85ab2c0efbd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/1b8e4c710dde46eebb81596a2961e853, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/38ffb2508ab041808cd81b6683617c79, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/756983697f9c47e6b35ba6eaa944112b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/4ded5e298d3045169fabbfffaae11f20, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f720290d2a1549cd9e06d754a6eae53e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bbb9961e37e2451d904202971a08234b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/8ab69c8dca5a41d880648127026c20ef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b3e4a34452984a4d94e0b5b7454bb499, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/851a60135ac8417992110082ecad9a5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/466d837b853a426bbdfd2eacdbe876c9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/47d269d6d60e458ba8de290060320d3e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f30ff97d206f4ba88b526b877155ac29, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/5abbdf2b6d42463fa78b493faeb2204a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b7bda0738918403f8e54ada17dc58089, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bf86f46b478445caa4075eedc5221ec3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/e9d7eb0c573947f1b337b0bab006e579, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/44c030f57faf4130a749e278f987143a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/53aa94687eda4d06a91fa96723597b0b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/489837606f7c41e7acfab4806a1c9236, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/439f4680f73a4572af9def55f8aa6346, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f5b435d624424ed294b4c7b76e5884fb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/022419fcfe1f46d2b0daee965ebf8925] to archive 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1008 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3024 rows 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1018 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1015 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3045 rows 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1015 2024-11-24T11:16:19,215 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3042 rows 2024-11-24T11:16:19,215 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:16:19,216 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:16:19,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:19,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:19,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:19,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:16:19,216 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-24T11:16:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:19,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/42d82f541bcc4f868f47e85ab2c0efbd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/42d82f541bcc4f868f47e85ab2c0efbd 2024-11-24T11:16:19,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/1b8e4c710dde46eebb81596a2961e853 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/1b8e4c710dde46eebb81596a2961e853 2024-11-24T11:16:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T11:16:19,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/38ffb2508ab041808cd81b6683617c79 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/38ffb2508ab041808cd81b6683617c79 2024-11-24T11:16:19,219 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446979219"}]},"ts":"1732446979219"} 2024-11-24T11:16:19,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/756983697f9c47e6b35ba6eaa944112b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/756983697f9c47e6b35ba6eaa944112b 2024-11-24T11:16:19,221 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T11:16:19,221 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T11:16:19,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/4ded5e298d3045169fabbfffaae11f20 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/4ded5e298d3045169fabbfffaae11f20 2024-11-24T11:16:19,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:16:19,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f720290d2a1549cd9e06d754a6eae53e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f720290d2a1549cd9e06d754a6eae53e 2024-11-24T11:16:19,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, UNASSIGN}] 2024-11-24T11:16:19,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bbb9961e37e2451d904202971a08234b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bbb9961e37e2451d904202971a08234b 2024-11-24T11:16:19,224 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, UNASSIGN 2024-11-24T11:16:19,224 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:19,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/8ab69c8dca5a41d880648127026c20ef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/8ab69c8dca5a41d880648127026c20ef 2024-11-24T11:16:19,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b3e4a34452984a4d94e0b5b7454bb499 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b3e4a34452984a4d94e0b5b7454bb499 2024-11-24T11:16:19,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, UNASSIGN because future has completed 2024-11-24T11:16:19,225 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:16:19,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:16:19,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/851a60135ac8417992110082ecad9a5b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/851a60135ac8417992110082ecad9a5b 2024-11-24T11:16:19,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/466d837b853a426bbdfd2eacdbe876c9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/466d837b853a426bbdfd2eacdbe876c9 2024-11-24T11:16:19,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/47d269d6d60e458ba8de290060320d3e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/47d269d6d60e458ba8de290060320d3e 2024-11-24T11:16:19,230 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f30ff97d206f4ba88b526b877155ac29 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f30ff97d206f4ba88b526b877155ac29 2024-11-24T11:16:19,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/5abbdf2b6d42463fa78b493faeb2204a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/5abbdf2b6d42463fa78b493faeb2204a 2024-11-24T11:16:19,232 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b7bda0738918403f8e54ada17dc58089 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/b7bda0738918403f8e54ada17dc58089 2024-11-24T11:16:19,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bf86f46b478445caa4075eedc5221ec3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/bf86f46b478445caa4075eedc5221ec3 2024-11-24T11:16:19,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/e9d7eb0c573947f1b337b0bab006e579 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/e9d7eb0c573947f1b337b0bab006e579 2024-11-24T11:16:19,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/44c030f57faf4130a749e278f987143a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/44c030f57faf4130a749e278f987143a 2024-11-24T11:16:19,236 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/53aa94687eda4d06a91fa96723597b0b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/53aa94687eda4d06a91fa96723597b0b 2024-11-24T11:16:19,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/489837606f7c41e7acfab4806a1c9236 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/489837606f7c41e7acfab4806a1c9236 2024-11-24T11:16:19,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/439f4680f73a4572af9def55f8aa6346 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/439f4680f73a4572af9def55f8aa6346 2024-11-24T11:16:19,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f5b435d624424ed294b4c7b76e5884fb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/f5b435d624424ed294b4c7b76e5884fb 2024-11-24T11:16:19,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/022419fcfe1f46d2b0daee965ebf8925 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/022419fcfe1f46d2b0daee965ebf8925 2024-11-24T11:16:19,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/44d62e8610bc447fad0f5b57940d81cd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1d99245b2f114a9998a92babbc34aef4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4b992afb263a4cbe86f09458aa6ce39f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/e6b4668c9d4c4587b9352607dfccca08, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/9a3f328ebc1b4a1aa32d6be217a14bfa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/5ee454898eb04a76bb8dc78792618640, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/58541420803349ca82e626bd15629f89, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/de0bb4db866e4694a78519e8d26dde5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/75e68c6769e34e1f95d5c7885eaf1f5f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/d9bdc2360d4d4c1cbb38a98f18706945, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/26774e0cb2444ca5ba56accfd2403802, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/3c600c953bd14435b68f9eaeb565719d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4c7b250d74d1433d88a2f75f9256770a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/dec25fb92e55453ba575000133e83a63, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/8f09eea37f484ebe9f8a4159cbf951ff, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1792fe7e7bfc4728ac8d8ee5be96b8eb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1a53abef6eec4fd2aa117f5e3fcda48a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/807678bc80f6448a87c1edaf1a7a65d9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4ab176f0decf441d8e1b9b50a3d9d79a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/314015879b23416ea24eb8171790edc7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0481016494b543c292b918b982d02e9b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/6b903854a40543c5a26cf31d5351ab16, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/833e75c5fd694b2692bdbfbe7353dc38] to archive 2024-11-24T11:16:19,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:16:19,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/44d62e8610bc447fad0f5b57940d81cd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/44d62e8610bc447fad0f5b57940d81cd 2024-11-24T11:16:19,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1d99245b2f114a9998a92babbc34aef4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1d99245b2f114a9998a92babbc34aef4 2024-11-24T11:16:19,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4b992afb263a4cbe86f09458aa6ce39f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4b992afb263a4cbe86f09458aa6ce39f 2024-11-24T11:16:19,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/e6b4668c9d4c4587b9352607dfccca08 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/e6b4668c9d4c4587b9352607dfccca08 2024-11-24T11:16:19,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/9a3f328ebc1b4a1aa32d6be217a14bfa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/9a3f328ebc1b4a1aa32d6be217a14bfa 2024-11-24T11:16:19,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/5ee454898eb04a76bb8dc78792618640 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/5ee454898eb04a76bb8dc78792618640 2024-11-24T11:16:19,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/58541420803349ca82e626bd15629f89 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/58541420803349ca82e626bd15629f89 2024-11-24T11:16:19,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/de0bb4db866e4694a78519e8d26dde5b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/de0bb4db866e4694a78519e8d26dde5b 2024-11-24T11:16:19,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/75e68c6769e34e1f95d5c7885eaf1f5f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/75e68c6769e34e1f95d5c7885eaf1f5f 2024-11-24T11:16:19,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/d9bdc2360d4d4c1cbb38a98f18706945 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/d9bdc2360d4d4c1cbb38a98f18706945 2024-11-24T11:16:19,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/26774e0cb2444ca5ba56accfd2403802 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/26774e0cb2444ca5ba56accfd2403802 2024-11-24T11:16:19,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/3c600c953bd14435b68f9eaeb565719d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/3c600c953bd14435b68f9eaeb565719d 2024-11-24T11:16:19,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4c7b250d74d1433d88a2f75f9256770a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4c7b250d74d1433d88a2f75f9256770a 2024-11-24T11:16:19,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/dec25fb92e55453ba575000133e83a63 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/dec25fb92e55453ba575000133e83a63 2024-11-24T11:16:19,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/8f09eea37f484ebe9f8a4159cbf951ff to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/8f09eea37f484ebe9f8a4159cbf951ff 2024-11-24T11:16:19,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1792fe7e7bfc4728ac8d8ee5be96b8eb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1792fe7e7bfc4728ac8d8ee5be96b8eb 2024-11-24T11:16:19,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1a53abef6eec4fd2aa117f5e3fcda48a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/1a53abef6eec4fd2aa117f5e3fcda48a 2024-11-24T11:16:19,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/807678bc80f6448a87c1edaf1a7a65d9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/807678bc80f6448a87c1edaf1a7a65d9 2024-11-24T11:16:19,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4ab176f0decf441d8e1b9b50a3d9d79a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/4ab176f0decf441d8e1b9b50a3d9d79a 2024-11-24T11:16:19,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/314015879b23416ea24eb8171790edc7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/314015879b23416ea24eb8171790edc7 2024-11-24T11:16:19,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0481016494b543c292b918b982d02e9b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0481016494b543c292b918b982d02e9b 2024-11-24T11:16:19,262 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/6b903854a40543c5a26cf31d5351ab16 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/6b903854a40543c5a26cf31d5351ab16 2024-11-24T11:16:19,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/833e75c5fd694b2692bdbfbe7353dc38 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/833e75c5fd694b2692bdbfbe7353dc38 2024-11-24T11:16:19,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/7806a2884cb94dbd9736dfe2a08ac8e0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/5147aa8921024717b0f15529a18d2125, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d6dea9b2c18a4b359ac5124f11f0a553, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/14c21458d18a4e9a817923f7d06cfccf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/26415018d72e473da0b3f9d60d4bd8f2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d521e53077924b08b576e628fb202a2a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/33305dca617248529cbe346cb744b7aa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/3128a50661d3440fbabca741dd1d552d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f5c4d2f806a4464984b2e756fc8448ea, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8fd18fcb971149899169c802d985993f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/6b32db082bc743eabda461131c212e14, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/93ec61873fa147c7a625758db61a8134, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/301c90c67ed04e64b5d24bdd15a71d5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/a013709057af44539808d9d98d91a81d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/78e57cb7f67843debb11a01acddd8e71, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/e8fbe74785ec4a36ba911db4858140b1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/786b1bcb27354913b352c19e485c16b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f9e3db7dab104c389918dbe30eb54fd2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f1acdb2e560b481aaa5406cceac01b59, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/863e0450bee6467595dd926d32621f55, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8a0322ce44c84512b998cb7745f6c8d4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/9f2078c2764e4398a7705069f3c2f774, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe] to archive 2024-11-24T11:16:19,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:16:19,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/7806a2884cb94dbd9736dfe2a08ac8e0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/7806a2884cb94dbd9736dfe2a08ac8e0 2024-11-24T11:16:19,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/5147aa8921024717b0f15529a18d2125 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/5147aa8921024717b0f15529a18d2125 2024-11-24T11:16:19,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d6dea9b2c18a4b359ac5124f11f0a553 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d6dea9b2c18a4b359ac5124f11f0a553 2024-11-24T11:16:19,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/14c21458d18a4e9a817923f7d06cfccf to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/14c21458d18a4e9a817923f7d06cfccf 2024-11-24T11:16:19,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/26415018d72e473da0b3f9d60d4bd8f2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/26415018d72e473da0b3f9d60d4bd8f2 2024-11-24T11:16:19,273 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d521e53077924b08b576e628fb202a2a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d521e53077924b08b576e628fb202a2a 2024-11-24T11:16:19,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/33305dca617248529cbe346cb744b7aa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/33305dca617248529cbe346cb744b7aa 2024-11-24T11:16:19,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/3128a50661d3440fbabca741dd1d552d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/3128a50661d3440fbabca741dd1d552d 2024-11-24T11:16:19,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f5c4d2f806a4464984b2e756fc8448ea to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f5c4d2f806a4464984b2e756fc8448ea 2024-11-24T11:16:19,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8fd18fcb971149899169c802d985993f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8fd18fcb971149899169c802d985993f 2024-11-24T11:16:19,277 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/6b32db082bc743eabda461131c212e14 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/6b32db082bc743eabda461131c212e14 2024-11-24T11:16:19,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/93ec61873fa147c7a625758db61a8134 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/93ec61873fa147c7a625758db61a8134 2024-11-24T11:16:19,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/301c90c67ed04e64b5d24bdd15a71d5b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/301c90c67ed04e64b5d24bdd15a71d5b 2024-11-24T11:16:19,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/a013709057af44539808d9d98d91a81d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/a013709057af44539808d9d98d91a81d 2024-11-24T11:16:19,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/78e57cb7f67843debb11a01acddd8e71 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/78e57cb7f67843debb11a01acddd8e71 2024-11-24T11:16:19,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/e8fbe74785ec4a36ba911db4858140b1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/e8fbe74785ec4a36ba911db4858140b1 2024-11-24T11:16:19,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/786b1bcb27354913b352c19e485c16b6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/786b1bcb27354913b352c19e485c16b6 2024-11-24T11:16:19,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f9e3db7dab104c389918dbe30eb54fd2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f9e3db7dab104c389918dbe30eb54fd2 2024-11-24T11:16:19,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f1acdb2e560b481aaa5406cceac01b59 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/f1acdb2e560b481aaa5406cceac01b59 2024-11-24T11:16:19,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/863e0450bee6467595dd926d32621f55 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/863e0450bee6467595dd926d32621f55 2024-11-24T11:16:19,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8a0322ce44c84512b998cb7745f6c8d4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/8a0322ce44c84512b998cb7745f6c8d4 2024-11-24T11:16:19,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/9f2078c2764e4398a7705069f3c2f774 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/9f2078c2764e4398a7705069f3c2f774 2024-11-24T11:16:19,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f5f73e76a839:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d5c9c6cdd9ec44a4b5732c79ed6d25fe 2024-11-24T11:16:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T11:16:19,378 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:19,378 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:16:19,378 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing f5f0024ce88b8047201380866bbadec0, disabling compactions & flushes 2024-11-24T11:16:19,378 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:19,378 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:19,378 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. after waiting 0 ms 2024-11-24T11:16:19,378 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:19,378 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing f5f0024ce88b8047201380866bbadec0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T11:16:19,378 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=A 2024-11-24T11:16:19,379 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:19,379 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=B 2024-11-24T11:16:19,379 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:19,379 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5f0024ce88b8047201380866bbadec0, store=C 2024-11-24T11:16:19,379 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:19,383 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242b80e0f6c8d7457e907c86dc49b97fdd_f5f0024ce88b8047201380866bbadec0 is 50, key is test_row_0/A:col10/1732446976824/Put/seqid=0 2024-11-24T11:16:19,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742258_1434 (size=12454) 2024-11-24T11:16:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T11:16:19,787 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:19,794 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242b80e0f6c8d7457e907c86dc49b97fdd_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242b80e0f6c8d7457e907c86dc49b97fdd_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:19,796 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/24896a7d0e404b328fd269b51ab9f6b7, store: [table=TestAcidGuarantees family=A region=f5f0024ce88b8047201380866bbadec0] 2024-11-24T11:16:19,797 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/24896a7d0e404b328fd269b51ab9f6b7 is 175, key is test_row_0/A:col10/1732446976824/Put/seqid=0 2024-11-24T11:16:19,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742259_1435 (size=31255) 2024-11-24T11:16:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T11:16:20,202 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/24896a7d0e404b328fd269b51ab9f6b7 2024-11-24T11:16:20,215 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/f0c464ae4ef04ef0b0d9593b955af1b8 is 50, key is test_row_0/B:col10/1732446976824/Put/seqid=0 2024-11-24T11:16:20,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742260_1436 (size=12301) 2024-11-24T11:16:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T11:16:20,621 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/f0c464ae4ef04ef0b0d9593b955af1b8 2024-11-24T11:16:20,629 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d31226b62be74eb7a2cd0477040c565a is 50, key is test_row_0/C:col10/1732446976824/Put/seqid=0 2024-11-24T11:16:20,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742261_1437 (size=12301) 2024-11-24T11:16:21,035 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d31226b62be74eb7a2cd0477040c565a 2024-11-24T11:16:21,042 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/A/24896a7d0e404b328fd269b51ab9f6b7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/24896a7d0e404b328fd269b51ab9f6b7 2024-11-24T11:16:21,048 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/24896a7d0e404b328fd269b51ab9f6b7, entries=150, sequenceid=354, filesize=30.5 K 2024-11-24T11:16:21,049 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/B/f0c464ae4ef04ef0b0d9593b955af1b8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/f0c464ae4ef04ef0b0d9593b955af1b8 2024-11-24T11:16:21,053 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/f0c464ae4ef04ef0b0d9593b955af1b8, entries=150, sequenceid=354, filesize=12.0 K 2024-11-24T11:16:21,054 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/.tmp/C/d31226b62be74eb7a2cd0477040c565a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d31226b62be74eb7a2cd0477040c565a 2024-11-24T11:16:21,057 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d31226b62be74eb7a2cd0477040c565a, entries=150, sequenceid=354, filesize=12.0 K 2024-11-24T11:16:21,058 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f5f0024ce88b8047201380866bbadec0 in 1680ms, sequenceid=354, compaction requested=true 2024-11-24T11:16:21,062 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/recovered.edits/357.seqid, newMaxSeqId=357, maxSeqId=4 2024-11-24T11:16:21,062 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. 2024-11-24T11:16:21,062 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for f5f0024ce88b8047201380866bbadec0: Waiting for close lock at 1732446979378Running coprocessor pre-close hooks at 1732446979378Disabling compacts and flushes for region at 1732446979378Disabling writes for close at 1732446979378Obtaining lock to block concurrent updates at 1732446979378Preparing flush snapshotting stores in f5f0024ce88b8047201380866bbadec0 at 1732446979378Finished memstore snapshotting TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732446979379 (+1 ms)Flushing stores of TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0. at 1732446979379Flushing f5f0024ce88b8047201380866bbadec0/A: creating writer at 1732446979379Flushing mob file f5f0024ce88b8047201380866bbadec0/A: appending metadata at 1732446979383 (+4 ms)Flushing mob file f5f0024ce88b8047201380866bbadec0/A: closing flushed file at 1732446979383Flushing f5f0024ce88b8047201380866bbadec0/A: appending metadata at 1732446979796 (+413 ms)Flushing f5f0024ce88b8047201380866bbadec0/A: closing flushed file at 1732446979796Flushing f5f0024ce88b8047201380866bbadec0/B: creating writer at 1732446980210 (+414 ms)Flushing f5f0024ce88b8047201380866bbadec0/B: appending metadata at 1732446980214 (+4 ms)Flushing f5f0024ce88b8047201380866bbadec0/B: closing flushed file at 1732446980214Flushing f5f0024ce88b8047201380866bbadec0/C: creating writer at 1732446980626 (+412 ms)Flushing f5f0024ce88b8047201380866bbadec0/C: appending metadata at 1732446980629 (+3 ms)Flushing f5f0024ce88b8047201380866bbadec0/C: closing flushed file at 1732446980629Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@588119d5: reopening flushed file at 1732446981041 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f51685d: reopening flushed file at 1732446981048 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a23e9ee: reopening flushed file at 1732446981053 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f5f0024ce88b8047201380866bbadec0 in 1680ms, sequenceid=354, compaction requested=true at 1732446981058 (+5 ms)Writing region close event to WAL at 1732446981059 (+1 ms)Running coprocessor post-close hooks at 1732446981062 (+3 ms)Closed at 1732446981062 2024-11-24T11:16:21,064 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,064 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=f5f0024ce88b8047201380866bbadec0, regionState=CLOSED 2024-11-24T11:16:21,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:16:21,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-24T11:16:21,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure f5f0024ce88b8047201380866bbadec0, server=f5f73e76a839,45939,1732446856176 in 1.8420 sec 2024-11-24T11:16:21,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-24T11:16:21,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5f0024ce88b8047201380866bbadec0, UNASSIGN in 1.8460 sec 2024-11-24T11:16:21,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-24T11:16:21,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8510 sec 2024-11-24T11:16:21,076 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446981076"}]},"ts":"1732446981076"} 2024-11-24T11:16:21,078 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T11:16:21,078 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T11:16:21,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8620 sec 2024-11-24T11:16:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T11:16:21,364 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:21,365 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-24T11:16:21,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:21,368 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T11:16:21,369 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:21,372 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,376 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/recovered.edits] 2024-11-24T11:16:21,380 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/24896a7d0e404b328fd269b51ab9f6b7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/24896a7d0e404b328fd269b51ab9f6b7 2024-11-24T11:16:21,382 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/50d9760b36a3435d81712c242d7bb437 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/50d9760b36a3435d81712c242d7bb437 2024-11-24T11:16:21,384 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/9730fc8261504a5a95360d1a1cb070a1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/A/9730fc8261504a5a95360d1a1cb070a1 2024-11-24T11:16:21,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0e34990cb0bb43b9a5aa8c939dda2610 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/0e34990cb0bb43b9a5aa8c939dda2610 2024-11-24T11:16:21,389 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/2b067cbbab6742d99979b1373df436e8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/2b067cbbab6742d99979b1373df436e8 2024-11-24T11:16:21,391 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/f0c464ae4ef04ef0b0d9593b955af1b8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/B/f0c464ae4ef04ef0b0d9593b955af1b8 2024-11-24T11:16:21,395 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/421048acf20b4007bf8c1077199477db to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/421048acf20b4007bf8c1077199477db 2024-11-24T11:16:21,396 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/623e18adfae6478a9c3298f5da965b30 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/623e18adfae6478a9c3298f5da965b30 2024-11-24T11:16:21,398 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d31226b62be74eb7a2cd0477040c565a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/C/d31226b62be74eb7a2cd0477040c565a 2024-11-24T11:16:21,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/recovered.edits/357.seqid to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0/recovered.edits/357.seqid 2024-11-24T11:16:21,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,400 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T11:16:21,400 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T11:16:21,401 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T11:16:21,403 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240cc8e55bdd484f75869dfba9c3bce231_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240cc8e55bdd484f75869dfba9c3bce231_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241b912693c4c14f419abb581d6a9966d1_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241b912693c4c14f419abb581d6a9966d1_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241ee6e53cea914bfe84d1a9966cfd1987_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241ee6e53cea914bfe84d1a9966cfd1987_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,406 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112421e568c977ac4be59427917cac76d144_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112421e568c977ac4be59427917cac76d144_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,407 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242a03e879a864472281a5274f6fa3db3b_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242a03e879a864472281a5274f6fa3db3b_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,408 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242b80e0f6c8d7457e907c86dc49b97fdd_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242b80e0f6c8d7457e907c86dc49b97fdd_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,409 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242ec62d7bb27d44548dcdb755a74a5adc_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242ec62d7bb27d44548dcdb755a74a5adc_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,410 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244a87fe02ed984bda89e03dbba0686241_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244a87fe02ed984bda89e03dbba0686241_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,411 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124588bad5180d147a2a45a6e0f097ac045_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124588bad5180d147a2a45a6e0f097ac045_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,412 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a7ee36f1fda49cc9a5b401de0882f18_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a7ee36f1fda49cc9a5b401de0882f18_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,413 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124828f5db87d8a4069b0f18842e4d1bd0d_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124828f5db87d8a4069b0f18842e4d1bd0d_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,414 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248489737f58cb403292d0f528022ee3f0_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248489737f58cb403292d0f528022ee3f0_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,415 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248be806d450c546449f0d38749f5071b9_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248be806d450c546449f0d38749f5071b9_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,416 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e5e828cdfe24426583f790f6b81fd861_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e5e828cdfe24426583f790f6b81fd861_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,417 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124eb261c875ced43acb308222cf0086cd1_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124eb261c875ced43acb308222cf0086cd1_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,418 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ee31212c5ddf4457a9119d679c8ccb5e_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ee31212c5ddf4457a9119d679c8ccb5e_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,419 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ef94232e708b4bb09728ed6f49fff5d5_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ef94232e708b4bb09728ed6f49fff5d5_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,420 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f6797dc7b6fb4d3baa87f555da32a3aa_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f6797dc7b6fb4d3baa87f555da32a3aa_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,421 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fca4e5ede7e648d2bbd63a9c16b42bf2_f5f0024ce88b8047201380866bbadec0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fca4e5ede7e648d2bbd63a9c16b42bf2_f5f0024ce88b8047201380866bbadec0 2024-11-24T11:16:21,421 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T11:16:21,423 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:21,425 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T11:16:21,426 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T11:16:21,427 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:21,427 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T11:16:21,428 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732446981427"}]},"ts":"9223372036854775807"} 2024-11-24T11:16:21,429 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T11:16:21,430 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f5f0024ce88b8047201380866bbadec0, NAME => 'TestAcidGuarantees,,1732446952799.f5f0024ce88b8047201380866bbadec0.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T11:16:21,430 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T11:16:21,430 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732446981430"}]},"ts":"9223372036854775807"} 2024-11-24T11:16:21,431 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T11:16:21,432 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:21,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-24T11:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T11:16:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T11:16:21,474 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:21,482 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=378 (was 350) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6125 (was 6849) 2024-11-24T11:16:21,493 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=378, ProcessCount=11, AvailableMemoryMB=6124 2024-11-24T11:16:21,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:16:21,495 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:16:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:21,497 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T11:16:21,497 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:21,498 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-24T11:16:21,498 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T11:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-24T11:16:21,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742262_1438 (size=1061) 2024-11-24T11:16:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-24T11:16:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-24T11:16:21,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:16:21,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742263_1439 (size=53) 2024-11-24T11:16:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-24T11:16:22,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:16:22,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 58548ec0d65011347d1619ef7b0b7484, disabling compactions & flushes 2024-11-24T11:16:22,318 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,318 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. after waiting 0 ms 2024-11-24T11:16:22,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,319 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 58548ec0d65011347d1619ef7b0b7484: Waiting for close lock at 1732446982318Disabling compacts and flushes for region at 1732446982318Disabling writes for close at 1732446982319 (+1 ms)Writing region close event to WAL at 1732446982319Closed at 1732446982319 2024-11-24T11:16:22,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T11:16:22,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732446982321"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732446982321"}]},"ts":"1732446982321"} 2024-11-24T11:16:22,322 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T11:16:22,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T11:16:22,324 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446982323"}]},"ts":"1732446982323"} 2024-11-24T11:16:22,325 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T11:16:22,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, ASSIGN}] 2024-11-24T11:16:22,326 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, ASSIGN 2024-11-24T11:16:22,327 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, ASSIGN; state=OFFLINE, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=false 2024-11-24T11:16:22,478 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=58548ec0d65011347d1619ef7b0b7484, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:22,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, ASSIGN because future has completed 2024-11-24T11:16:22,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:16:22,633 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-24T11:16:22,633 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:16:22,633 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,633 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:16:22,634 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,634 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,635 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,636 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:16:22,636 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58548ec0d65011347d1619ef7b0b7484 columnFamilyName A 2024-11-24T11:16:22,636 DEBUG [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:22,636 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.HStore(327): Store=58548ec0d65011347d1619ef7b0b7484/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:16:22,636 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,637 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:16:22,638 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58548ec0d65011347d1619ef7b0b7484 columnFamilyName B 2024-11-24T11:16:22,638 DEBUG [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:22,638 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.HStore(327): Store=58548ec0d65011347d1619ef7b0b7484/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:16:22,638 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,640 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:16:22,640 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58548ec0d65011347d1619ef7b0b7484 columnFamilyName C 2024-11-24T11:16:22,640 DEBUG [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:22,640 INFO [StoreOpener-58548ec0d65011347d1619ef7b0b7484-1 {}] regionserver.HStore(327): Store=58548ec0d65011347d1619ef7b0b7484/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:16:22,640 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,640 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,641 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,641 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,642 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,642 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,642 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:16:22,643 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,645 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:16:22,645 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 58548ec0d65011347d1619ef7b0b7484; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65105339, jitterRate=-0.029854848980903625}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:16:22,645 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:22,646 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 58548ec0d65011347d1619ef7b0b7484: Running coprocessor pre-open hook at 1732446982634Writing region info on filesystem at 1732446982634Initializing all the Stores at 1732446982634Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446982634Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446982634Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732446982635 (+1 ms)Cleaning up temporary data from old regions at 1732446982642 (+7 ms)Running coprocessor post-open hooks at 1732446982645 (+3 ms)Region opened successfully at 1732446982646 (+1 ms) 2024-11-24T11:16:22,646 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., pid=110, masterSystemTime=1732446982631 2024-11-24T11:16:22,648 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,648 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:22,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=58548ec0d65011347d1619ef7b0b7484, regionState=OPEN, openSeqNum=2, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:22,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:16:22,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-24T11:16:22,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 in 171 msec 2024-11-24T11:16:22,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-24T11:16:22,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, ASSIGN in 327 msec 2024-11-24T11:16:22,655 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T11:16:22,655 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732446982655"}]},"ts":"1732446982655"} 2024-11-24T11:16:22,657 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T11:16:22,658 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T11:16:22,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-24T11:16:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-24T11:16:23,654 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:23,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54872992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bec75e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae60179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,667 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,671 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T11:16:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f5f73e76a839,40345,1732446855400 2024-11-24T11:16:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fab361e 2024-11-24T11:16:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T11:16:23,674 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T11:16:23,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514e87d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,678 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2c960a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,681 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3423a0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250ddf0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ad6724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c03de7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc2be37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635dbb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,692 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390e74a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c47751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a9425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,697 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2828653e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e1154a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3fa1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513e71f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6735, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,703 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b017c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,705 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e8c44e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19aa15e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb66e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,710 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31704df1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f3cd49e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a148f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e483c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b78dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,721 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314e9b88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,723 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b732091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c5ed93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:23,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:23,734 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47179f36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:23,736 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7010faa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:23,740 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,741 DEBUG [Thread-1980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,746 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,748 DEBUG [Thread-1984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,749 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,749 DEBUG [Thread-1976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,750 DEBUG [Thread-1993 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,750 DEBUG [Thread-1993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,752 DEBUG [Thread-1978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,750 DEBUG [Thread-1984 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,753 DEBUG [Thread-1978 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,753 DEBUG [Thread-1978 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,753 DEBUG [Thread-1984 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:23,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:23,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:23,768 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,772 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,773 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:23,774 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:23,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-24T11:16:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T11:16:23,778 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:23,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:23,782 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:23,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2] 2024-11-24T11:16:23,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447043792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447043790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447043801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446983786, max timestamp=1732446983801Num uniques 859; 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446983786, max timestamp=1732446983801Num uniques 859; 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:23,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:16:23,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:16:23,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732446983786, max timestamp=1732446983801Num uniques 859; 2024-11-24T11:16:23,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:16:23,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/658302356ecb4373a0689ac4dd2bd0b6 is 50, key is test_row_0/A:col10/1732446983757/Put/seqid=0 2024-11-24T11:16:23,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742264_1440 (size=12001) 2024-11-24T11:16:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T11:16:23,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T11:16:23,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:23,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447043947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447043966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447043969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447043969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447043973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447044084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T11:16:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T11:16:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447044104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447044104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447044104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447044104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T11:16:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/658302356ecb4373a0689ac4dd2bd0b6 2024-11-24T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/254ac1b052cc4423bd2aa4cf1524c762 is 50, key is test_row_0/B:col10/1732446983757/Put/seqid=0 2024-11-24T11:16:24,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447044293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742265_1441 (size=12001) 2024-11-24T11:16:24,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447044324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447044325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447044326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447044333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T11:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:24,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T11:16:24,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T11:16:24,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447044614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447044644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447044644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447044644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447044653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:24,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T11:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:24,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/254ac1b052cc4423bd2aa4cf1524c762 2024-11-24T11:16:24,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/377a5374a6c948c9ae3c4bb5df2950b0 is 50, key is test_row_0/C:col10/1732446983757/Put/seqid=0 2024-11-24T11:16:24,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742266_1442 (size=12001) 2024-11-24T11:16:24,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/377a5374a6c948c9ae3c4bb5df2950b0 2024-11-24T11:16:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/658302356ecb4373a0689ac4dd2bd0b6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/658302356ecb4373a0689ac4dd2bd0b6 2024-11-24T11:16:24,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/658302356ecb4373a0689ac4dd2bd0b6, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T11:16:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/254ac1b052cc4423bd2aa4cf1524c762 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/254ac1b052cc4423bd2aa4cf1524c762 2024-11-24T11:16:24,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/254ac1b052cc4423bd2aa4cf1524c762, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T11:16:24,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/377a5374a6c948c9ae3c4bb5df2950b0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/377a5374a6c948c9ae3c4bb5df2950b0 2024-11-24T11:16:24,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/377a5374a6c948c9ae3c4bb5df2950b0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T11:16:24,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 58548ec0d65011347d1619ef7b0b7484 in 1065ms, sequenceid=12, compaction requested=false 2024-11-24T11:16:24,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:24,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T11:16:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:24,864 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T11:16:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:16:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:16:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:16:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b40bc000085a449b9bfd9177a95a3d6d is 50, key is test_row_0/A:col10/1732446983964/Put/seqid=0 2024-11-24T11:16:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T11:16:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742267_1443 (size=12001) 2024-11-24T11:16:25,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447045191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447045192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447045195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447045197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447045198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447045303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,319 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b40bc000085a449b9bfd9177a95a3d6d 2024-11-24T11:16:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/cc49023b62ec4c31a5c1115fa03e2063 is 50, key is test_row_0/B:col10/1732446983964/Put/seqid=0 2024-11-24T11:16:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447045324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447045324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447045324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447045324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742268_1444 (size=12001) 2024-11-24T11:16:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447045524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447045533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447045533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447045533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447045533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,731 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/cc49023b62ec4c31a5c1115fa03e2063 2024-11-24T11:16:25,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/07107716e70f4a2b95aa74735561b3f5 is 50, key is test_row_0/C:col10/1732446983964/Put/seqid=0 2024-11-24T11:16:25,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742269_1445 (size=12001) 2024-11-24T11:16:25,740 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/07107716e70f4a2b95aa74735561b3f5 2024-11-24T11:16:25,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b40bc000085a449b9bfd9177a95a3d6d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b40bc000085a449b9bfd9177a95a3d6d 2024-11-24T11:16:25,748 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b40bc000085a449b9bfd9177a95a3d6d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T11:16:25,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/cc49023b62ec4c31a5c1115fa03e2063 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/cc49023b62ec4c31a5c1115fa03e2063 2024-11-24T11:16:25,755 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/cc49023b62ec4c31a5c1115fa03e2063, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T11:16:25,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/07107716e70f4a2b95aa74735561b3f5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07107716e70f4a2b95aa74735561b3f5 2024-11-24T11:16:25,760 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07107716e70f4a2b95aa74735561b3f5, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T11:16:25,760 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 58548ec0d65011347d1619ef7b0b7484 in 896ms, sequenceid=48, compaction requested=false 2024-11-24T11:16:25,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:25,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:25,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-24T11:16:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-24T11:16:25,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-24T11:16:25,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9800 sec 2024-11-24T11:16:25,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9910 sec 2024-11-24T11:16:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:25,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T11:16:25,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:25,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/83a4810f0dc64d8985a87923e4e8b6c1 is 50, key is test_row_0/A:col10/1732446985849/Put/seqid=0 2024-11-24T11:16:25,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742270_1446 (size=19021) 2024-11-24T11:16:25,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/83a4810f0dc64d8985a87923e4e8b6c1 2024-11-24T11:16:25,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/7ca21586489c4b4b8169569d6ff8d2e2 is 50, key is test_row_0/B:col10/1732446985849/Put/seqid=0 2024-11-24T11:16:25,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742271_1447 (size=12001) 2024-11-24T11:16:25,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/7ca21586489c4b4b8169569d6ff8d2e2 2024-11-24T11:16:25,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/16de1b8a1a5e4d808b83bb3d5482a492 is 50, key is test_row_0/C:col10/1732446985849/Put/seqid=0 2024-11-24T11:16:25,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742272_1448 (size=12001) 2024-11-24T11:16:25,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/16de1b8a1a5e4d808b83bb3d5482a492 2024-11-24T11:16:25,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/83a4810f0dc64d8985a87923e4e8b6c1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/83a4810f0dc64d8985a87923e4e8b6c1 2024-11-24T11:16:25,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447045886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447045886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447045888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447045890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/83a4810f0dc64d8985a87923e4e8b6c1, entries=300, sequenceid=62, filesize=18.6 K 2024-11-24T11:16:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/7ca21586489c4b4b8169569d6ff8d2e2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7ca21586489c4b4b8169569d6ff8d2e2 2024-11-24T11:16:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447045902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:25,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7ca21586489c4b4b8169569d6ff8d2e2, entries=150, sequenceid=62, filesize=11.7 K 2024-11-24T11:16:25,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/16de1b8a1a5e4d808b83bb3d5482a492 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/16de1b8a1a5e4d808b83bb3d5482a492 2024-11-24T11:16:25,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/16de1b8a1a5e4d808b83bb3d5482a492, entries=150, sequenceid=62, filesize=11.7 K 2024-11-24T11:16:25,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 58548ec0d65011347d1619ef7b0b7484 in 71ms, sequenceid=62, compaction requested=true 2024-11-24T11:16:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:25,923 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:25,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T11:16:25,924 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T11:16:25,925 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:25,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:25,926 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:25,926 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:25,926 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:25,926 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/658302356ecb4373a0689ac4dd2bd0b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b40bc000085a449b9bfd9177a95a3d6d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/83a4810f0dc64d8985a87923e4e8b6c1] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=42.0 K 2024-11-24T11:16:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-24T11:16:25,926 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:25,926 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:25,926 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:25,927 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/254ac1b052cc4423bd2aa4cf1524c762, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/cc49023b62ec4c31a5c1115fa03e2063, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7ca21586489c4b4b8169569d6ff8d2e2] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=35.2 K 2024-11-24T11:16:25,927 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 658302356ecb4373a0689ac4dd2bd0b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446983757 2024-11-24T11:16:25,927 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T11:16:25,929 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:25,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:25,933 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254ac1b052cc4423bd2aa4cf1524c762, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446983757 2024-11-24T11:16:25,933 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b40bc000085a449b9bfd9177a95a3d6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732446983862 2024-11-24T11:16:25,933 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 83a4810f0dc64d8985a87923e4e8b6c1, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446985188 2024-11-24T11:16:25,934 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc49023b62ec4c31a5c1115fa03e2063, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732446983862 2024-11-24T11:16:25,934 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca21586489c4b4b8169569d6ff8d2e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446985849 2024-11-24T11:16:25,953 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:25,953 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/74911627db3d4b04a36d8041de048881 is 50, key is test_row_0/A:col10/1732446985849/Put/seqid=0 2024-11-24T11:16:25,967 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:25,967 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/ce7222c5eb7c4770b7b4536973f8bd8d is 50, key is test_row_0/B:col10/1732446985849/Put/seqid=0 2024-11-24T11:16:26,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742273_1449 (size=12104) 2024-11-24T11:16:26,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742274_1450 (size=12104) 2024-11-24T11:16:26,014 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/74911627db3d4b04a36d8041de048881 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/74911627db3d4b04a36d8041de048881 2024-11-24T11:16:26,024 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into 74911627db3d4b04a36d8041de048881(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:26,024 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:26,024 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732446985923; duration=0sec 2024-11-24T11:16:26,024 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:26,024 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:26,024 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:26,025 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:26,025 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:26,025 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:26,025 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/377a5374a6c948c9ae3c4bb5df2950b0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07107716e70f4a2b95aa74735561b3f5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/16de1b8a1a5e4d808b83bb3d5482a492] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=35.2 K 2024-11-24T11:16:26,027 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 377a5374a6c948c9ae3c4bb5df2950b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732446983757 2024-11-24T11:16:26,028 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 07107716e70f4a2b95aa74735561b3f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732446983862 2024-11-24T11:16:26,028 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 16de1b8a1a5e4d808b83bb3d5482a492, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446985849 2024-11-24T11:16:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:26,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:16:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T11:16:26,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/26abf4977cac41139b4dd5dc15a8be1c is 50, key is test_row_0/A:col10/1732446985886/Put/seqid=0 2024-11-24T11:16:26,039 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:26,039 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/efd3b7211a20455d912862911e6a2345 is 50, key is test_row_0/C:col10/1732446985849/Put/seqid=0 2024-11-24T11:16:26,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T11:16:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:26,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:26,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447046062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447046068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447046069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742276_1452 (size=14341) 2024-11-24T11:16:26,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/26abf4977cac41139b4dd5dc15a8be1c 2024-11-24T11:16:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447046077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742275_1451 (size=12104) 2024-11-24T11:16:26,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b02ba69ff686444795703f449d2331a6 is 50, key is test_row_0/B:col10/1732446985886/Put/seqid=0 2024-11-24T11:16:26,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447046095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742277_1453 (size=12001) 2024-11-24T11:16:26,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b02ba69ff686444795703f449d2331a6 2024-11-24T11:16:26,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/c1ea4b4dc5c14874ad14eb1524af10c6 is 50, key is test_row_0/C:col10/1732446985886/Put/seqid=0 2024-11-24T11:16:26,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447046206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447046206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447046206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447046213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742278_1454 (size=12001) 2024-11-24T11:16:26,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/c1ea4b4dc5c14874ad14eb1524af10c6 2024-11-24T11:16:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/26abf4977cac41139b4dd5dc15a8be1c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/26abf4977cac41139b4dd5dc15a8be1c 2024-11-24T11:16:26,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T11:16:26,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:26,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:26,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T11:16:26,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/26abf4977cac41139b4dd5dc15a8be1c, entries=200, sequenceid=82, filesize=14.0 K 2024-11-24T11:16:26,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b02ba69ff686444795703f449d2331a6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b02ba69ff686444795703f449d2331a6 2024-11-24T11:16:26,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b02ba69ff686444795703f449d2331a6, entries=150, sequenceid=82, filesize=11.7 K 2024-11-24T11:16:26,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447046234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/c1ea4b4dc5c14874ad14eb1524af10c6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/c1ea4b4dc5c14874ad14eb1524af10c6 2024-11-24T11:16:26,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/c1ea4b4dc5c14874ad14eb1524af10c6, entries=150, sequenceid=82, filesize=11.7 K 2024-11-24T11:16:26,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 58548ec0d65011347d1619ef7b0b7484 in 228ms, sequenceid=82, compaction requested=false 2024-11-24T11:16:26,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:26,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T11:16:26,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:26,394 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T11:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:26,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca is 50, key is test_row_0/A:col10/1732446986069/Put/seqid=0 2024-11-24T11:16:26,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742279_1455 (size=12001) 2024-11-24T11:16:26,418 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/ce7222c5eb7c4770b7b4536973f8bd8d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/ce7222c5eb7c4770b7b4536973f8bd8d 2024-11-24T11:16:26,422 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into ce7222c5eb7c4770b7b4536973f8bd8d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:26,422 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:26,422 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732446985923; duration=0sec 2024-11-24T11:16:26,422 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:26,422 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:26,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:26,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447046457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447046458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447046463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447046466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447046467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,507 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/efd3b7211a20455d912862911e6a2345 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/efd3b7211a20455d912862911e6a2345 2024-11-24T11:16:26,511 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into efd3b7211a20455d912862911e6a2345(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:26,511 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:26,511 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732446985923; duration=0sec 2024-11-24T11:16:26,511 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:26,511 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T11:16:26,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447046573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447046573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447046574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447046574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,600 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T11:16:26,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447046773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447046783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447046784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447046784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447046784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:26,802 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca 2024-11-24T11:16:26,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/5324719602e74d2491aa54c9c0eb4d21 is 50, key is test_row_0/B:col10/1732446986069/Put/seqid=0 2024-11-24T11:16:26,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742280_1456 (size=12001) 2024-11-24T11:16:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T11:16:27,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447047093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447047093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447047094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447047094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,213 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/5324719602e74d2491aa54c9c0eb4d21 2024-11-24T11:16:27,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/8b5b6478346e4ce08210624e33a70221 is 50, key is test_row_0/C:col10/1732446986069/Put/seqid=0 2024-11-24T11:16:27,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742281_1457 (size=12001) 2024-11-24T11:16:27,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447047283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,622 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/8b5b6478346e4ce08210624e33a70221 2024-11-24T11:16:27,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447047605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447047614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447047614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447047614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:27,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca 2024-11-24T11:16:27,665 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T11:16:27,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/5324719602e74d2491aa54c9c0eb4d21 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/5324719602e74d2491aa54c9c0eb4d21 2024-11-24T11:16:27,672 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/5324719602e74d2491aa54c9c0eb4d21, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T11:16:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/8b5b6478346e4ce08210624e33a70221 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/8b5b6478346e4ce08210624e33a70221 2024-11-24T11:16:27,677 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/8b5b6478346e4ce08210624e33a70221, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T11:16:27,678 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 58548ec0d65011347d1619ef7b0b7484 in 1284ms, sequenceid=95, compaction requested=true 2024-11-24T11:16:27,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:27,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-24T11:16:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-24T11:16:27,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-24T11:16:27,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7540 sec 2024-11-24T11:16:27,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.7620 sec 2024-11-24T11:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T11:16:28,074 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:28,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-24T11:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T11:16:28,077 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:28,080 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:28,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T11:16:28,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T11:16:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:28,233 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T11:16:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:28,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/f12be9bde8374fdabe656a1c43b0dedd is 50, key is test_row_0/A:col10/1732446986452/Put/seqid=0 2024-11-24T11:16:28,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742282_1458 (size=12001) 2024-11-24T11:16:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447048354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T11:16:28,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447048463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:28,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:28,640 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/f12be9bde8374fdabe656a1c43b0dedd 2024-11-24T11:16:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447048633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:28,641 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:28,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:28,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f7ff6d871c8a4aff9b2f2ce648225520 is 50, key is test_row_0/B:col10/1732446986452/Put/seqid=0 2024-11-24T11:16:28,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742283_1459 (size=12001) 2024-11-24T11:16:28,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447048644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:28,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447048644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:28,657 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:28,657 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:28,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447048655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:28,668 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:28,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447048673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T11:16:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447048994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:29,050 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f7ff6d871c8a4aff9b2f2ce648225520 2024-11-24T11:16:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/5caa9cbaee314b848df56d8bc65a8d1f is 50, key is test_row_0/C:col10/1732446986452/Put/seqid=0 2024-11-24T11:16:29,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742284_1460 (size=12001) 2024-11-24T11:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T11:16:29,459 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/5caa9cbaee314b848df56d8bc65a8d1f 2024-11-24T11:16:29,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/f12be9bde8374fdabe656a1c43b0dedd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/f12be9bde8374fdabe656a1c43b0dedd 2024-11-24T11:16:29,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/f12be9bde8374fdabe656a1c43b0dedd, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T11:16:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f7ff6d871c8a4aff9b2f2ce648225520 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7ff6d871c8a4aff9b2f2ce648225520 2024-11-24T11:16:29,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7ff6d871c8a4aff9b2f2ce648225520, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T11:16:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/5caa9cbaee314b848df56d8bc65a8d1f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/5caa9cbaee314b848df56d8bc65a8d1f 2024-11-24T11:16:29,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/5caa9cbaee314b848df56d8bc65a8d1f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T11:16:29,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 58548ec0d65011347d1619ef7b0b7484 in 1262ms, sequenceid=117, compaction requested=true 2024-11-24T11:16:29,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:29,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:29,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-24T11:16:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-24T11:16:29,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-24T11:16:29,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-24T11:16:29,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4270 sec 2024-11-24T11:16:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:29,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T11:16:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:29,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/bf890057972b4f6ab4f435c3b4507f4c is 50, key is test_row_0/A:col10/1732446988353/Put/seqid=0 2024-11-24T11:16:29,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742285_1461 (size=14391) 2024-11-24T11:16:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447049819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:29,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447049943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:30,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/bf890057972b4f6ab4f435c3b4507f4c 2024-11-24T11:16:30,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/07f033d732ad46759ecc2e0560b0af6f is 50, key is test_row_0/B:col10/1732446988353/Put/seqid=0 2024-11-24T11:16:30,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742286_1462 (size=12051) 2024-11-24T11:16:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447050164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T11:16:30,224 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:30,224 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-24T11:16:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T11:16:30,225 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:30,226 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:30,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T11:16:30,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T11:16:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/07f033d732ad46759ecc2e0560b0af6f 2024-11-24T11:16:30,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/7608ea678bd6486785c4bb6225d98c4a is 50, key is test_row_0/C:col10/1732446988353/Put/seqid=0 2024-11-24T11:16:30,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742287_1463 (size=12051) 2024-11-24T11:16:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447050484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:30,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T11:16:30,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:30,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T11:16:30,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447050663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:30,675 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:30,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447050673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:30,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447050673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:30,677 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,678 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:30,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T11:16:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447050684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:30,691 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:30,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/7608ea678bd6486785c4bb6225d98c4a 2024-11-24T11:16:30,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/bf890057972b4f6ab4f435c3b4507f4c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/bf890057972b4f6ab4f435c3b4507f4c 2024-11-24T11:16:30,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/bf890057972b4f6ab4f435c3b4507f4c, entries=200, sequenceid=130, filesize=14.1 K 2024-11-24T11:16:30,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/07f033d732ad46759ecc2e0560b0af6f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/07f033d732ad46759ecc2e0560b0af6f 2024-11-24T11:16:30,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/07f033d732ad46759ecc2e0560b0af6f, entries=150, sequenceid=130, filesize=11.8 K 2024-11-24T11:16:30,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T11:16:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/7608ea678bd6486785c4bb6225d98c4a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/7608ea678bd6486785c4bb6225d98c4a 2024-11-24T11:16:30,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:30,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/7608ea678bd6486785c4bb6225d98c4a, entries=150, sequenceid=130, filesize=11.8 K 2024-11-24T11:16:30,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 58548ec0d65011347d1619ef7b0b7484 in 1306ms, sequenceid=130, compaction requested=true 2024-11-24T11:16:30,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:30,842 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T11:16:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:30,842 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T11:16:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:30,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:30,843 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T11:16:30,843 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64838 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T11:16:30,843 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:30,843 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:30,843 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,843 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,843 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/74911627db3d4b04a36d8041de048881, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/26abf4977cac41139b4dd5dc15a8be1c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/f12be9bde8374fdabe656a1c43b0dedd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/bf890057972b4f6ab4f435c3b4507f4c] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=63.3 K 2024-11-24T11:16:30,843 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/ce7222c5eb7c4770b7b4536973f8bd8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b02ba69ff686444795703f449d2331a6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/5324719602e74d2491aa54c9c0eb4d21, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7ff6d871c8a4aff9b2f2ce648225520, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/07f033d732ad46759ecc2e0560b0af6f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=58.7 K 2024-11-24T11:16:30,843 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7222c5eb7c4770b7b4536973f8bd8d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446985849 2024-11-24T11:16:30,843 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74911627db3d4b04a36d8041de048881, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446985849 2024-11-24T11:16:30,844 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b02ba69ff686444795703f449d2331a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732446985879 2024-11-24T11:16:30,844 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26abf4977cac41139b4dd5dc15a8be1c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732446985879 2024-11-24T11:16:30,844 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 5324719602e74d2491aa54c9c0eb4d21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732446986062 2024-11-24T11:16:30,844 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cdaf3811e1b4ec5a18dcc142a6ed6ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732446986062 2024-11-24T11:16:30,844 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12be9bde8374fdabe656a1c43b0dedd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732446986452 2024-11-24T11:16:30,844 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ff6d871c8a4aff9b2f2ce648225520, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732446986452 2024-11-24T11:16:30,845 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf890057972b4f6ab4f435c3b4507f4c, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732446988340 2024-11-24T11:16:30,845 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f033d732ad46759ecc2e0560b0af6f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732446988340 2024-11-24T11:16:30,852 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:30,853 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/306921faa6c7434099faf87982455696 is 50, key is test_row_0/A:col10/1732446988353/Put/seqid=0 2024-11-24T11:16:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T11:16:30,856 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#388 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:30,857 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a2bd07afdce045d6b66255074e1180bd is 50, key is test_row_0/B:col10/1732446988353/Put/seqid=0 2024-11-24T11:16:30,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742288_1464 (size=12325) 2024-11-24T11:16:30,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742289_1465 (size=12325) 2024-11-24T11:16:30,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T11:16:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:30,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:16:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:30,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/859460d62a4c416bacf65e2132223e5f is 50, key is test_row_0/A:col10/1732446989781/Put/seqid=0 2024-11-24T11:16:31,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742290_1466 (size=12151) 2024-11-24T11:16:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:31,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:31,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447051100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447051213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:31,265 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/306921faa6c7434099faf87982455696 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/306921faa6c7434099faf87982455696 2024-11-24T11:16:31,267 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a2bd07afdce045d6b66255074e1180bd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a2bd07afdce045d6b66255074e1180bd 2024-11-24T11:16:31,268 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into 306921faa6c7434099faf87982455696(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:31,268 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:31,268 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=11, startTime=1732446990841; duration=0sec 2024-11-24T11:16:31,268 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:31,268 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:31,268 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T11:16:31,269 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T11:16:31,269 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:31,269 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:31,269 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/efd3b7211a20455d912862911e6a2345, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/c1ea4b4dc5c14874ad14eb1524af10c6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/8b5b6478346e4ce08210624e33a70221, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/5caa9cbaee314b848df56d8bc65a8d1f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/7608ea678bd6486785c4bb6225d98c4a] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=58.7 K 2024-11-24T11:16:31,270 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd3b7211a20455d912862911e6a2345, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732446985849 2024-11-24T11:16:31,270 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ea4b4dc5c14874ad14eb1524af10c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732446985879 2024-11-24T11:16:31,270 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b5b6478346e4ce08210624e33a70221, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732446986062 2024-11-24T11:16:31,270 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5caa9cbaee314b848df56d8bc65a8d1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732446986452 2024-11-24T11:16:31,271 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7608ea678bd6486785c4bb6225d98c4a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732446988340 2024-11-24T11:16:31,271 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into a2bd07afdce045d6b66255074e1180bd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:31,271 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:31,271 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=11, startTime=1732446990842; duration=0sec 2024-11-24T11:16:31,271 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:31,271 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:31,278 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#390 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:31,278 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/fe3bb2079dfb44ac8531cfc3f664b2ad is 50, key is test_row_0/C:col10/1732446988353/Put/seqid=0 2024-11-24T11:16:31,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742291_1467 (size=12325) 2024-11-24T11:16:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T11:16:31,401 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/859460d62a4c416bacf65e2132223e5f 2024-11-24T11:16:31,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/9b6c2f1211ef4ab0a7053a0f11105183 is 50, key is test_row_0/B:col10/1732446989781/Put/seqid=0 2024-11-24T11:16:31,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742292_1468 (size=12151) 2024-11-24T11:16:31,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447051423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:31,685 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/fe3bb2079dfb44ac8531cfc3f664b2ad as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/fe3bb2079dfb44ac8531cfc3f664b2ad 2024-11-24T11:16:31,688 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into fe3bb2079dfb44ac8531cfc3f664b2ad(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:31,688 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:31,688 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=11, startTime=1732446990842; duration=0sec 2024-11-24T11:16:31,688 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:31,688 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:31,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447051743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:31,810 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/9b6c2f1211ef4ab0a7053a0f11105183 2024-11-24T11:16:31,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/d27a44055a734ce39f1f4f345d72fa8d is 50, key is test_row_0/C:col10/1732446989781/Put/seqid=0 2024-11-24T11:16:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742293_1469 (size=12151) 2024-11-24T11:16:32,221 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/d27a44055a734ce39f1f4f345d72fa8d 2024-11-24T11:16:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/859460d62a4c416bacf65e2132223e5f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/859460d62a4c416bacf65e2132223e5f 2024-11-24T11:16:32,229 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/859460d62a4c416bacf65e2132223e5f, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T11:16:32,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/9b6c2f1211ef4ab0a7053a0f11105183 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/9b6c2f1211ef4ab0a7053a0f11105183 2024-11-24T11:16:32,233 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/9b6c2f1211ef4ab0a7053a0f11105183, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T11:16:32,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/d27a44055a734ce39f1f4f345d72fa8d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d27a44055a734ce39f1f4f345d72fa8d 2024-11-24T11:16:32,236 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d27a44055a734ce39f1f4f345d72fa8d, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T11:16:32,237 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 58548ec0d65011347d1619ef7b0b7484 in 1248ms, sequenceid=149, compaction requested=false 2024-11-24T11:16:32,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:32,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-24T11:16:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-24T11:16:32,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-24T11:16:32,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0130 sec 2024-11-24T11:16:32,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.0160 sec 2024-11-24T11:16:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:32,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:16:32,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:32,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:32,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:32,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:32,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:32,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:32,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b139015db12a4b708448a2859b9042bd is 50, key is test_row_0/A:col10/1732446991098/Put/seqid=0 2024-11-24T11:16:32,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742294_1470 (size=14541) 2024-11-24T11:16:32,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b139015db12a4b708448a2859b9042bd 2024-11-24T11:16:32,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/7df75acc55a1402dafcb6adc3222684d is 50, key is test_row_0/B:col10/1732446991098/Put/seqid=0 2024-11-24T11:16:32,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742295_1471 (size=12151) 2024-11-24T11:16:32,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447052352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T11:16:32,384 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:32,384 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-24T11:16:32,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:32,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:32,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T11:16:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447052464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:32,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:32,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:32,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T11:16:32,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T11:16:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:32,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/7df75acc55a1402dafcb6adc3222684d 2024-11-24T11:16:32,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f0fcdcf757074425bd93e5d27714ceee is 50, key is test_row_0/C:col10/1732446991098/Put/seqid=0 2024-11-24T11:16:32,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447052683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:32,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742296_1472 (size=12151) 2024-11-24T11:16:32,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f0fcdcf757074425bd93e5d27714ceee 2024-11-24T11:16:32,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T11:16:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:32,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b139015db12a4b708448a2859b9042bd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b139015db12a4b708448a2859b9042bd 2024-11-24T11:16:32,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b139015db12a4b708448a2859b9042bd, entries=200, sequenceid=166, filesize=14.2 K 2024-11-24T11:16:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/7df75acc55a1402dafcb6adc3222684d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7df75acc55a1402dafcb6adc3222684d 2024-11-24T11:16:32,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7df75acc55a1402dafcb6adc3222684d, entries=150, sequenceid=166, filesize=11.9 K 2024-11-24T11:16:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f0fcdcf757074425bd93e5d27714ceee as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0fcdcf757074425bd93e5d27714ceee 2024-11-24T11:16:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T11:16:32,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0fcdcf757074425bd93e5d27714ceee, entries=150, sequenceid=166, filesize=11.9 K 2024-11-24T11:16:32,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 58548ec0d65011347d1619ef7b0b7484 in 449ms, sequenceid=166, compaction requested=true 2024-11-24T11:16:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:32,709 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:32,710 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:32,711 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39017 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:32,711 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:32,712 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,712 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/306921faa6c7434099faf87982455696, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/859460d62a4c416bacf65e2132223e5f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b139015db12a4b708448a2859b9042bd] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=38.1 K 2024-11-24T11:16:32,713 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 306921faa6c7434099faf87982455696, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732446988340 2024-11-24T11:16:32,713 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:32,713 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:32,713 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,714 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a2bd07afdce045d6b66255074e1180bd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/9b6c2f1211ef4ab0a7053a0f11105183, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7df75acc55a1402dafcb6adc3222684d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=35.8 K 2024-11-24T11:16:32,714 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 859460d62a4c416bacf65e2132223e5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446989781 2024-11-24T11:16:32,714 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a2bd07afdce045d6b66255074e1180bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732446988340 2024-11-24T11:16:32,714 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b139015db12a4b708448a2859b9042bd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732446991049 2024-11-24T11:16:32,714 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b6c2f1211ef4ab0a7053a0f11105183, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446989781 2024-11-24T11:16:32,715 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df75acc55a1402dafcb6adc3222684d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732446991049 2024-11-24T11:16:32,724 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:32,724 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/9e57a493890a472e94db83cc60a7d038 is 50, key is test_row_0/A:col10/1732446991098/Put/seqid=0 2024-11-24T11:16:32,724 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:32,728 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/72ef2c8a20a340ec850f1d24afdcdf9b is 50, key is test_row_0/B:col10/1732446991098/Put/seqid=0 2024-11-24T11:16:32,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742297_1473 (size=12527) 2024-11-24T11:16:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742298_1474 (size=12527) 2024-11-24T11:16:32,738 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/9e57a493890a472e94db83cc60a7d038 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9e57a493890a472e94db83cc60a7d038 2024-11-24T11:16:32,743 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/72ef2c8a20a340ec850f1d24afdcdf9b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/72ef2c8a20a340ec850f1d24afdcdf9b 2024-11-24T11:16:32,745 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into 9e57a493890a472e94db83cc60a7d038(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:32,745 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:32,745 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732446992709; duration=0sec 2024-11-24T11:16:32,745 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:32,746 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:32,746 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:32,748 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:32,748 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:32,748 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,748 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/fe3bb2079dfb44ac8531cfc3f664b2ad, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d27a44055a734ce39f1f4f345d72fa8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0fcdcf757074425bd93e5d27714ceee] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=35.8 K 2024-11-24T11:16:32,749 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3bb2079dfb44ac8531cfc3f664b2ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732446988340 2024-11-24T11:16:32,749 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d27a44055a734ce39f1f4f345d72fa8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732446989781 2024-11-24T11:16:32,749 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0fcdcf757074425bd93e5d27714ceee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732446991049 2024-11-24T11:16:32,751 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into 72ef2c8a20a340ec850f1d24afdcdf9b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:32,751 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:32,751 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732446992709; duration=0sec 2024-11-24T11:16:32,751 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:32,751 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:32,756 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:32,756 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/4247a007f67d4d4d86f69029e086a672 is 50, key is test_row_0/C:col10/1732446991098/Put/seqid=0 2024-11-24T11:16:32,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742299_1475 (size=12527) 2024-11-24T11:16:32,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T11:16:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:32,844 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:16:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:32,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:32,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:32,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:32,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/55a4d841b0e145f78fe033dcb70c1a9a is 50, key is test_row_0/A:col10/1732446992326/Put/seqid=0 2024-11-24T11:16:32,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742300_1476 (size=12151) 2024-11-24T11:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:32,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T11:16:33,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447053092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:33,175 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/4247a007f67d4d4d86f69029e086a672 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/4247a007f67d4d4d86f69029e086a672 2024-11-24T11:16:33,179 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into 4247a007f67d4d4d86f69029e086a672(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:33,179 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:33,179 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732446992710; duration=0sec 2024-11-24T11:16:33,179 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:33,179 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:33,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447053214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:33,263 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/55a4d841b0e145f78fe033dcb70c1a9a 2024-11-24T11:16:33,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b4d610e3e0c0448e9313e7b139edbfd6 is 50, key is test_row_0/B:col10/1732446992326/Put/seqid=0 2024-11-24T11:16:33,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742301_1477 (size=12151) 2024-11-24T11:16:33,314 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b4d610e3e0c0448e9313e7b139edbfd6 2024-11-24T11:16:33,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/62cbe063e3184fadb6760bfd8eccb899 is 50, key is test_row_0/C:col10/1732446992326/Put/seqid=0 2024-11-24T11:16:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742302_1478 (size=12151) 2024-11-24T11:16:33,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447053433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T11:16:33,730 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/62cbe063e3184fadb6760bfd8eccb899 2024-11-24T11:16:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/55a4d841b0e145f78fe033dcb70c1a9a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/55a4d841b0e145f78fe033dcb70c1a9a 2024-11-24T11:16:33,741 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/55a4d841b0e145f78fe033dcb70c1a9a, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T11:16:33,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b4d610e3e0c0448e9313e7b139edbfd6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b4d610e3e0c0448e9313e7b139edbfd6 2024-11-24T11:16:33,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b4d610e3e0c0448e9313e7b139edbfd6, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T11:16:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/62cbe063e3184fadb6760bfd8eccb899 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/62cbe063e3184fadb6760bfd8eccb899 2024-11-24T11:16:33,747 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/62cbe063e3184fadb6760bfd8eccb899, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T11:16:33,748 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 58548ec0d65011347d1619ef7b0b7484 in 904ms, sequenceid=186, compaction requested=false 2024-11-24T11:16:33,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:33,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:33,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-24T11:16:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-24T11:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:33,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:16:33,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:33,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:33,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:33,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:33,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:33,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:33,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-24T11:16:33,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3630 sec 2024-11-24T11:16:33,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.3680 sec 2024-11-24T11:16:33,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/12eea9b3f79642d18ade35f1d45fb5c4 is 50, key is test_row_0/A:col10/1732446993749/Put/seqid=0 2024-11-24T11:16:33,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742303_1479 (size=14541) 2024-11-24T11:16:33,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447053837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:33,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:33,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447053954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:34,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/12eea9b3f79642d18ade35f1d45fb5c4 2024-11-24T11:16:34,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/163d8a4e97b9425c96474898fd39b8ab is 50, key is test_row_0/B:col10/1732446993749/Put/seqid=0 2024-11-24T11:16:34,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742304_1480 (size=12151) 2024-11-24T11:16:34,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447054174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447054484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T11:16:34,533 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:34,534 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-24T11:16:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T11:16:34,536 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:34,536 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:34,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:34,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/163d8a4e97b9425c96474898fd39b8ab 2024-11-24T11:16:34,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/1e1766d7988447908f8e312095efeee9 is 50, key is test_row_0/C:col10/1732446993749/Put/seqid=0 2024-11-24T11:16:34,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742305_1481 (size=12151) 2024-11-24T11:16:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T11:16:34,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T11:16:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:34,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447054693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447054694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:34,697 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:34,697 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447054713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447054713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:34,720 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:34,721 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:34,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T11:16:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:34,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T11:16:34,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/1e1766d7988447908f8e312095efeee9 2024-11-24T11:16:34,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/12eea9b3f79642d18ade35f1d45fb5c4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/12eea9b3f79642d18ade35f1d45fb5c4 2024-11-24T11:16:34,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/12eea9b3f79642d18ade35f1d45fb5c4, entries=200, sequenceid=202, filesize=14.2 K 2024-11-24T11:16:34,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/163d8a4e97b9425c96474898fd39b8ab as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/163d8a4e97b9425c96474898fd39b8ab 2024-11-24T11:16:34,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/163d8a4e97b9425c96474898fd39b8ab, entries=150, sequenceid=202, filesize=11.9 K 2024-11-24T11:16:34,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/1e1766d7988447908f8e312095efeee9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/1e1766d7988447908f8e312095efeee9 2024-11-24T11:16:34,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/1e1766d7988447908f8e312095efeee9, entries=150, sequenceid=202, filesize=11.9 K 2024-11-24T11:16:34,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 58548ec0d65011347d1619ef7b0b7484 in 1243ms, sequenceid=202, compaction requested=true 2024-11-24T11:16:34,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:34,993 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:34,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:34,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:34,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:34,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:34,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:34,993 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:34,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:34,994 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:34,994 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:34,994 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:34,994 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:34,994 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:34,994 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9e57a493890a472e94db83cc60a7d038, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/55a4d841b0e145f78fe033dcb70c1a9a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/12eea9b3f79642d18ade35f1d45fb5c4] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=38.3 K 2024-11-24T11:16:34,994 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:34,994 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/72ef2c8a20a340ec850f1d24afdcdf9b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b4d610e3e0c0448e9313e7b139edbfd6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/163d8a4e97b9425c96474898fd39b8ab] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.0 K 2024-11-24T11:16:34,994 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e57a493890a472e94db83cc60a7d038, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732446991049 2024-11-24T11:16:34,994 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ef2c8a20a340ec850f1d24afdcdf9b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732446991049 2024-11-24T11:16:34,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T11:16:34,994 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d610e3e0c0448e9313e7b139edbfd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446992326 2024-11-24T11:16:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:34,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:16:34,995 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 163d8a4e97b9425c96474898fd39b8ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732446993084 2024-11-24T11:16:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:34,996 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55a4d841b0e145f78fe033dcb70c1a9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446992326 2024-11-24T11:16:34,996 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12eea9b3f79642d18ade35f1d45fb5c4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732446993044 2024-11-24T11:16:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/157adc329062435099668847a9cb6e0a is 50, key is test_row_0/A:col10/1732446993814/Put/seqid=0 2024-11-24T11:16:35,000 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:35,000 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b2367044622d44c4acfcc5de0bf34eef is 50, key is test_row_0/B:col10/1732446993749/Put/seqid=0 2024-11-24T11:16:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:35,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:35,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742306_1482 (size=12151) 2024-11-24T11:16:35,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742307_1483 (size=12629) 2024-11-24T11:16:35,012 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:35,012 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/aa9e8bf277804f3d99b48e6856ddebb1 is 50, key is test_row_0/A:col10/1732446993749/Put/seqid=0 2024-11-24T11:16:35,016 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/b2367044622d44c4acfcc5de0bf34eef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b2367044622d44c4acfcc5de0bf34eef 2024-11-24T11:16:35,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742308_1484 (size=12629) 2024-11-24T11:16:35,020 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into b2367044622d44c4acfcc5de0bf34eef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:35,020 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:35,020 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732446994993; duration=0sec 2024-11-24T11:16:35,020 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:35,021 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:35,021 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:35,021 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:35,021 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:35,021 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:35,022 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/4247a007f67d4d4d86f69029e086a672, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/62cbe063e3184fadb6760bfd8eccb899, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/1e1766d7988447908f8e312095efeee9] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.0 K 2024-11-24T11:16:35,022 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4247a007f67d4d4d86f69029e086a672, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732446991049 2024-11-24T11:16:35,022 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 62cbe063e3184fadb6760bfd8eccb899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732446992326 2024-11-24T11:16:35,022 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e1766d7988447908f8e312095efeee9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732446993084 2024-11-24T11:16:35,030 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:35,030 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/551cfe742328447f949ac5976747089e is 50, key is test_row_0/C:col10/1732446993749/Put/seqid=0 2024-11-24T11:16:35,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742309_1485 (size=12629) 2024-11-24T11:16:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447055090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T11:16:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447055203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:35,406 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/157adc329062435099668847a9cb6e0a 2024-11-24T11:16:35,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f69a81e9bfe7450fbfd199ec0552c8c0 is 50, key is test_row_0/B:col10/1732446993814/Put/seqid=0 2024-11-24T11:16:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742310_1486 (size=12151) 2024-11-24T11:16:35,420 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/aa9e8bf277804f3d99b48e6856ddebb1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa9e8bf277804f3d99b48e6856ddebb1 2024-11-24T11:16:35,423 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into aa9e8bf277804f3d99b48e6856ddebb1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:35,423 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:35,423 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732446994993; duration=0sec 2024-11-24T11:16:35,423 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:35,423 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447055423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:35,437 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/551cfe742328447f949ac5976747089e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/551cfe742328447f949ac5976747089e 2024-11-24T11:16:35,440 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into 551cfe742328447f949ac5976747089e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:35,440 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:35,440 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732446994993; duration=0sec 2024-11-24T11:16:35,440 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:35,440 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T11:16:35,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447055744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:35,815 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f69a81e9bfe7450fbfd199ec0552c8c0 2024-11-24T11:16:35,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/b29f4e956a374bf985fc67a88ece2cb5 is 50, key is test_row_0/C:col10/1732446993814/Put/seqid=0 2024-11-24T11:16:35,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742311_1487 (size=12151) 2024-11-24T11:16:36,225 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/b29f4e956a374bf985fc67a88ece2cb5 2024-11-24T11:16:36,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/157adc329062435099668847a9cb6e0a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/157adc329062435099668847a9cb6e0a 2024-11-24T11:16:36,233 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/157adc329062435099668847a9cb6e0a, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T11:16:36,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f69a81e9bfe7450fbfd199ec0552c8c0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f69a81e9bfe7450fbfd199ec0552c8c0 2024-11-24T11:16:36,237 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f69a81e9bfe7450fbfd199ec0552c8c0, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T11:16:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/b29f4e956a374bf985fc67a88ece2cb5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b29f4e956a374bf985fc67a88ece2cb5 2024-11-24T11:16:36,240 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b29f4e956a374bf985fc67a88ece2cb5, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T11:16:36,241 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 58548ec0d65011347d1619ef7b0b7484 in 1247ms, sequenceid=219, compaction requested=false 2024-11-24T11:16:36,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:36,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:36,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-24T11:16:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-24T11:16:36,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-24T11:16:36,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7060 sec 2024-11-24T11:16:36,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.7100 sec 2024-11-24T11:16:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:36,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:16:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:36,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ac071d7cd1be44ac9434c19fe13eac5b is 50, key is test_row_0/A:col10/1732446995088/Put/seqid=0 2024-11-24T11:16:36,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742312_1488 (size=14541) 2024-11-24T11:16:36,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447056376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:36,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447056493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:36,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ac071d7cd1be44ac9434c19fe13eac5b 2024-11-24T11:16:36,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a3950ecea219461c829728d9dc5e5b8d is 50, key is test_row_0/B:col10/1732446995088/Put/seqid=0 2024-11-24T11:16:36,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742313_1489 (size=12151) 2024-11-24T11:16:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T11:16:36,684 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:36,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-24T11:16:36,686 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T11:16:36,688 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:36,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:36,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447056713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T11:16:36,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T11:16:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:36,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:36,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T11:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:36,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T11:16:37,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447057023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:37,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a3950ecea219461c829728d9dc5e5b8d 2024-11-24T11:16:37,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f0ec4d2a1b834bd1a3340b7a51962503 is 50, key is test_row_0/C:col10/1732446995088/Put/seqid=0 2024-11-24T11:16:37,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742314_1490 (size=12151) 2024-11-24T11:16:37,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f0ec4d2a1b834bd1a3340b7a51962503 2024-11-24T11:16:37,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ac071d7cd1be44ac9434c19fe13eac5b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ac071d7cd1be44ac9434c19fe13eac5b 2024-11-24T11:16:37,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ac071d7cd1be44ac9434c19fe13eac5b, entries=200, sequenceid=238, filesize=14.2 K 2024-11-24T11:16:37,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a3950ecea219461c829728d9dc5e5b8d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a3950ecea219461c829728d9dc5e5b8d 2024-11-24T11:16:37,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a3950ecea219461c829728d9dc5e5b8d, entries=150, sequenceid=238, filesize=11.9 K 2024-11-24T11:16:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f0ec4d2a1b834bd1a3340b7a51962503 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0ec4d2a1b834bd1a3340b7a51962503 2024-11-24T11:16:37,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0ec4d2a1b834bd1a3340b7a51962503, entries=150, sequenceid=238, filesize=11.9 K 2024-11-24T11:16:37,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 58548ec0d65011347d1619ef7b0b7484 in 854ms, sequenceid=238, compaction requested=true 2024-11-24T11:16:37,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:37,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:37,116 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:37,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:37,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:37,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:37,116 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:37,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:37,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:37,118 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:37,118 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:37,118 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:37,118 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:37,119 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:37,119 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:37,119 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b2367044622d44c4acfcc5de0bf34eef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f69a81e9bfe7450fbfd199ec0552c8c0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a3950ecea219461c829728d9dc5e5b8d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.1 K 2024-11-24T11:16:37,119 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa9e8bf277804f3d99b48e6856ddebb1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/157adc329062435099668847a9cb6e0a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ac071d7cd1be44ac9434c19fe13eac5b] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=38.4 K 2024-11-24T11:16:37,119 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b2367044622d44c4acfcc5de0bf34eef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732446993084 2024-11-24T11:16:37,119 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9e8bf277804f3d99b48e6856ddebb1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732446993084 2024-11-24T11:16:37,121 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f69a81e9bfe7450fbfd199ec0552c8c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732446993814 2024-11-24T11:16:37,121 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 157adc329062435099668847a9cb6e0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732446993814 2024-11-24T11:16:37,122 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a3950ecea219461c829728d9dc5e5b8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732446995080 2024-11-24T11:16:37,122 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac071d7cd1be44ac9434c19fe13eac5b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732446995064 2024-11-24T11:16:37,130 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:37,131 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/37cf9c92a5774e009f3e40a54916d521 is 50, key is test_row_0/B:col10/1732446995088/Put/seqid=0 2024-11-24T11:16:37,133 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:37,134 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/47a83467cc334839ad47c883aa978bd7 is 50, key is test_row_0/A:col10/1732446995088/Put/seqid=0 2024-11-24T11:16:37,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742315_1491 (size=12731) 2024-11-24T11:16:37,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742316_1492 (size=12731) 2024-11-24T11:16:37,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T11:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:37,147 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:37,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:37,151 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/37cf9c92a5774e009f3e40a54916d521 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/37cf9c92a5774e009f3e40a54916d521 2024-11-24T11:16:37,152 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/47a83467cc334839ad47c883aa978bd7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/47a83467cc334839ad47c883aa978bd7 2024-11-24T11:16:37,154 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into 37cf9c92a5774e009f3e40a54916d521(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:37,154 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:37,154 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732446997116; duration=0sec 2024-11-24T11:16:37,154 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:37,154 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:37,155 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:37,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/44f56587ddb4429c8963d4ce7da6ff65 is 50, key is test_row_0/A:col10/1732446996369/Put/seqid=0 2024-11-24T11:16:37,156 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into 47a83467cc334839ad47c883aa978bd7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:37,156 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:37,156 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732446997116; duration=0sec 2024-11-24T11:16:37,156 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:37,156 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:37,158 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:37,158 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:37,158 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:37,159 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/551cfe742328447f949ac5976747089e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b29f4e956a374bf985fc67a88ece2cb5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0ec4d2a1b834bd1a3340b7a51962503] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.1 K 2024-11-24T11:16:37,159 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 551cfe742328447f949ac5976747089e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732446993084 2024-11-24T11:16:37,159 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b29f4e956a374bf985fc67a88ece2cb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732446993814 2024-11-24T11:16:37,159 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f0ec4d2a1b834bd1a3340b7a51962503, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732446995080 2024-11-24T11:16:37,165 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:37,166 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/e0acc7c80a7a4fe999a647fc7368efa6 is 50, key is test_row_0/C:col10/1732446995088/Put/seqid=0 2024-11-24T11:16:37,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742317_1493 (size=12151) 2024-11-24T11:16:37,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742318_1494 (size=12731) 2024-11-24T11:16:37,171 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/44f56587ddb4429c8963d4ce7da6ff65 2024-11-24T11:16:37,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/53c8446971a24650a0cd01ef6636df21 is 50, key is test_row_0/B:col10/1732446996369/Put/seqid=0 2024-11-24T11:16:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742319_1495 (size=12151) 2024-11-24T11:16:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T11:16:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:37,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:37,577 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/e0acc7c80a7a4fe999a647fc7368efa6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e0acc7c80a7a4fe999a647fc7368efa6 2024-11-24T11:16:37,580 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into e0acc7c80a7a4fe999a647fc7368efa6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:37,580 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:37,580 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732446997116; duration=0sec 2024-11-24T11:16:37,581 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:37,581 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:37,581 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/53c8446971a24650a0cd01ef6636df21 2024-11-24T11:16:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/cad80cbc31ad4f319653a9504f692c90 is 50, key is test_row_0/C:col10/1732446996369/Put/seqid=0 2024-11-24T11:16:37,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742320_1496 (size=12151) 2024-11-24T11:16:37,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447057632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:37,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447057744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:37,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T11:16:37,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447057964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:37,992 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/cad80cbc31ad4f319653a9504f692c90 2024-11-24T11:16:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/44f56587ddb4429c8963d4ce7da6ff65 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/44f56587ddb4429c8963d4ce7da6ff65 2024-11-24T11:16:38,004 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/44f56587ddb4429c8963d4ce7da6ff65, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T11:16:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/53c8446971a24650a0cd01ef6636df21 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/53c8446971a24650a0cd01ef6636df21 2024-11-24T11:16:38,013 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/53c8446971a24650a0cd01ef6636df21, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T11:16:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/cad80cbc31ad4f319653a9504f692c90 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/cad80cbc31ad4f319653a9504f692c90 2024-11-24T11:16:38,020 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/cad80cbc31ad4f319653a9504f692c90, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T11:16:38,021 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 58548ec0d65011347d1619ef7b0b7484 in 874ms, sequenceid=254, compaction requested=false 2024-11-24T11:16:38,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:38,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:38,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-24T11:16:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-24T11:16:38,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-24T11:16:38,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3350 sec 2024-11-24T11:16:38,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3410 sec 2024-11-24T11:16:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:38,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:16:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:38,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:38,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:38,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:38,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:38,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:38,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ccec2d8d43244914a1356f999bcbc550 is 50, key is test_row_0/A:col10/1732446998286/Put/seqid=0 2024-11-24T11:16:38,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742321_1497 (size=14741) 2024-11-24T11:16:38,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ccec2d8d43244914a1356f999bcbc550 2024-11-24T11:16:38,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4c30b19a15e24d3a80fe4a820433ed0d is 50, key is test_row_0/B:col10/1732446998286/Put/seqid=0 2024-11-24T11:16:38,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742322_1498 (size=12301) 2024-11-24T11:16:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447058368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447058483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:38,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4c30b19a15e24d3a80fe4a820433ed0d 2024-11-24T11:16:38,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447058704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:38,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/d4a2a96402fc47e38a2ef7ab96a39b34 is 50, key is test_row_0/C:col10/1732446998286/Put/seqid=0 2024-11-24T11:16:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742323_1499 (size=12301) 2024-11-24T11:16:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T11:16:38,834 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:38,834 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-24T11:16:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T11:16:38,836 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:38,836 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:38,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T11:16:38,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T11:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:38,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:39,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447059023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:39,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/d4a2a96402fc47e38a2ef7ab96a39b34 2024-11-24T11:16:39,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ccec2d8d43244914a1356f999bcbc550 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ccec2d8d43244914a1356f999bcbc550 2024-11-24T11:16:39,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ccec2d8d43244914a1356f999bcbc550, entries=200, sequenceid=274, filesize=14.4 K 2024-11-24T11:16:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4c30b19a15e24d3a80fe4a820433ed0d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4c30b19a15e24d3a80fe4a820433ed0d 2024-11-24T11:16:39,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4c30b19a15e24d3a80fe4a820433ed0d, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T11:16:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/d4a2a96402fc47e38a2ef7ab96a39b34 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d4a2a96402fc47e38a2ef7ab96a39b34 2024-11-24T11:16:39,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d4a2a96402fc47e38a2ef7ab96a39b34, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T11:16:39,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 58548ec0d65011347d1619ef7b0b7484 in 847ms, sequenceid=274, compaction requested=true 2024-11-24T11:16:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:39,134 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:39,134 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:39,135 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:39,135 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:39,135 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/47a83467cc334839ad47c883aa978bd7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/44f56587ddb4429c8963d4ce7da6ff65, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ccec2d8d43244914a1356f999bcbc550] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=38.7 K 2024-11-24T11:16:39,135 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/37cf9c92a5774e009f3e40a54916d521, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/53c8446971a24650a0cd01ef6636df21, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4c30b19a15e24d3a80fe4a820433ed0d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.3 K 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 37cf9c92a5774e009f3e40a54916d521, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732446995080 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47a83467cc334839ad47c883aa978bd7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732446995080 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c8446971a24650a0cd01ef6636df21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446996349 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f56587ddb4429c8963d4ce7da6ff65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446996349 2024-11-24T11:16:39,135 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccec2d8d43244914a1356f999bcbc550, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732446997602 2024-11-24T11:16:39,136 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c30b19a15e24d3a80fe4a820433ed0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732446997602 2024-11-24T11:16:39,141 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:39,141 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/45c7e32ac02142beb4b976024c5cadd9 is 50, key is test_row_0/A:col10/1732446998286/Put/seqid=0 2024-11-24T11:16:39,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T11:16:39,142 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:39,142 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:16:39,142 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/fe423218ecf84bf18cb25ba545db23f2 is 50, key is test_row_0/B:col10/1732446998286/Put/seqid=0 2024-11-24T11:16:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:39,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/be6350de4bf44a72bd73a8aa5db55d79 is 50, key is test_row_0/A:col10/1732446998367/Put/seqid=0 2024-11-24T11:16:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742324_1500 (size=12983) 2024-11-24T11:16:39,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742325_1501 (size=12983) 2024-11-24T11:16:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T11:16:39,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742326_1502 (size=12301) 2024-11-24T11:16:39,160 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/be6350de4bf44a72bd73a8aa5db55d79 2024-11-24T11:16:39,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f7041753a67e4975b03ae3f5aa8aacca is 50, key is test_row_0/B:col10/1732446998367/Put/seqid=0 2024-11-24T11:16:39,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742327_1503 (size=12301) 2024-11-24T11:16:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T11:16:39,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:39,553 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/45c7e32ac02142beb4b976024c5cadd9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/45c7e32ac02142beb4b976024c5cadd9 2024-11-24T11:16:39,557 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/fe423218ecf84bf18cb25ba545db23f2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/fe423218ecf84bf18cb25ba545db23f2 2024-11-24T11:16:39,558 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into 45c7e32ac02142beb4b976024c5cadd9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:39,558 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:39,558 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732446999134; duration=0sec 2024-11-24T11:16:39,558 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:39,558 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:39,558 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:39,559 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:39,559 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:39,559 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:39,559 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e0acc7c80a7a4fe999a647fc7368efa6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/cad80cbc31ad4f319653a9504f692c90, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d4a2a96402fc47e38a2ef7ab96a39b34] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.3 K 2024-11-24T11:16:39,559 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0acc7c80a7a4fe999a647fc7368efa6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732446995080 2024-11-24T11:16:39,559 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad80cbc31ad4f319653a9504f692c90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732446996349 2024-11-24T11:16:39,560 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a2a96402fc47e38a2ef7ab96a39b34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732446997602 2024-11-24T11:16:39,561 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into fe423218ecf84bf18cb25ba545db23f2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:39,561 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:39,561 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732446999134; duration=0sec 2024-11-24T11:16:39,561 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:39,561 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:39,566 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:39,567 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/9275a6f7eb464360bfe0db18bd3b8411 is 50, key is test_row_0/C:col10/1732446998286/Put/seqid=0 2024-11-24T11:16:39,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742328_1504 (size=12983) 2024-11-24T11:16:39,571 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f7041753a67e4975b03ae3f5aa8aacca 2024-11-24T11:16:39,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/07649c572ef34c0db8ef57bde7167d37 is 50, key is test_row_0/C:col10/1732446998367/Put/seqid=0 2024-11-24T11:16:39,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742329_1505 (size=12301) 2024-11-24T11:16:39,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447059674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:39,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447059794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T11:16:39,974 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/9275a6f7eb464360bfe0db18bd3b8411 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/9275a6f7eb464360bfe0db18bd3b8411 2024-11-24T11:16:39,977 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into 9275a6f7eb464360bfe0db18bd3b8411(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:39,977 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:39,977 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732446999134; duration=0sec 2024-11-24T11:16:39,977 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:39,977 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:39,984 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/07649c572ef34c0db8ef57bde7167d37 2024-11-24T11:16:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/be6350de4bf44a72bd73a8aa5db55d79 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/be6350de4bf44a72bd73a8aa5db55d79 2024-11-24T11:16:39,991 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/be6350de4bf44a72bd73a8aa5db55d79, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T11:16:39,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/f7041753a67e4975b03ae3f5aa8aacca as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7041753a67e4975b03ae3f5aa8aacca 2024-11-24T11:16:39,994 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7041753a67e4975b03ae3f5aa8aacca, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T11:16:39,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/07649c572ef34c0db8ef57bde7167d37 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07649c572ef34c0db8ef57bde7167d37 2024-11-24T11:16:39,997 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07649c572ef34c0db8ef57bde7167d37, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T11:16:39,998 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 58548ec0d65011347d1619ef7b0b7484 in 856ms, sequenceid=289, compaction requested=false 2024-11-24T11:16:39,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:39,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:39,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-24T11:16:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-24T11:16:40,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-24T11:16:40,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1630 sec 2024-11-24T11:16:40,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.1670 sec 2024-11-24T11:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:40,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T11:16:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:40,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b04d9791ee3c489991035122ff05f81c is 50, key is test_row_0/A:col10/1732446999630/Put/seqid=0 2024-11-24T11:16:40,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742330_1506 (size=14741) 2024-11-24T11:16:40,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b04d9791ee3c489991035122ff05f81c 2024-11-24T11:16:40,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/784aa20baacb4ce6a05a48896df92a2c is 50, key is test_row_0/B:col10/1732446999630/Put/seqid=0 2024-11-24T11:16:40,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742331_1507 (size=12301) 2024-11-24T11:16:40,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447060081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:40,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447060193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:40,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447060404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:40,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/784aa20baacb4ce6a05a48896df92a2c 2024-11-24T11:16:40,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/32a9b125cf674b1fb848e6d151ed5a4d is 50, key is test_row_0/C:col10/1732446999630/Put/seqid=0 2024-11-24T11:16:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742332_1508 (size=12301) 2024-11-24T11:16:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447060714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:40,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/32a9b125cf674b1fb848e6d151ed5a4d 2024-11-24T11:16:40,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/b04d9791ee3c489991035122ff05f81c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b04d9791ee3c489991035122ff05f81c 2024-11-24T11:16:40,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b04d9791ee3c489991035122ff05f81c, entries=200, sequenceid=310, filesize=14.4 K 2024-11-24T11:16:40,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/784aa20baacb4ce6a05a48896df92a2c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/784aa20baacb4ce6a05a48896df92a2c 2024-11-24T11:16:40,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/784aa20baacb4ce6a05a48896df92a2c, entries=150, sequenceid=310, filesize=12.0 K 2024-11-24T11:16:40,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/32a9b125cf674b1fb848e6d151ed5a4d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/32a9b125cf674b1fb848e6d151ed5a4d 2024-11-24T11:16:40,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/32a9b125cf674b1fb848e6d151ed5a4d, entries=150, sequenceid=310, filesize=12.0 K 2024-11-24T11:16:40,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 58548ec0d65011347d1619ef7b0b7484 in 839ms, sequenceid=310, compaction requested=true 2024-11-24T11:16:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:40,847 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:40,847 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:40,848 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:40,848 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:40,848 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/fe423218ecf84bf18cb25ba545db23f2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7041753a67e4975b03ae3f5aa8aacca, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/784aa20baacb4ce6a05a48896df92a2c] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.7 K 2024-11-24T11:16:40,848 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/45c7e32ac02142beb4b976024c5cadd9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/be6350de4bf44a72bd73a8aa5db55d79, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b04d9791ee3c489991035122ff05f81c] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=39.1 K 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting fe423218ecf84bf18cb25ba545db23f2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732446997602 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c7e32ac02142beb4b976024c5cadd9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732446997602 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f7041753a67e4975b03ae3f5aa8aacca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446998331 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting be6350de4bf44a72bd73a8aa5db55d79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446998331 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 784aa20baacb4ce6a05a48896df92a2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732446999630 2024-11-24T11:16:40,848 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04d9791ee3c489991035122ff05f81c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732446999630 2024-11-24T11:16:40,854 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:40,854 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/2720635d50fc4384801c0b69676fd2d9 is 50, key is test_row_0/B:col10/1732446999630/Put/seqid=0 2024-11-24T11:16:40,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742333_1509 (size=13085) 2024-11-24T11:16:40,857 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:40,857 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/87eaf4e455c44e80ad512b44f2a115b8 is 50, key is test_row_0/A:col10/1732446999630/Put/seqid=0 2024-11-24T11:16:40,861 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/2720635d50fc4384801c0b69676fd2d9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/2720635d50fc4384801c0b69676fd2d9 2024-11-24T11:16:40,864 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into 2720635d50fc4384801c0b69676fd2d9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:40,864 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:40,864 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732447000847; duration=0sec 2024-11-24T11:16:40,864 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:40,864 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:40,864 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:40,865 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:40,865 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:40,865 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:40,865 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/9275a6f7eb464360bfe0db18bd3b8411, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07649c572ef34c0db8ef57bde7167d37, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/32a9b125cf674b1fb848e6d151ed5a4d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.7 K 2024-11-24T11:16:40,866 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9275a6f7eb464360bfe0db18bd3b8411, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732446997602 2024-11-24T11:16:40,866 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 07649c572ef34c0db8ef57bde7167d37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732446998331 2024-11-24T11:16:40,866 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a9b125cf674b1fb848e6d151ed5a4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732446999630 2024-11-24T11:16:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742334_1510 (size=13085) 2024-11-24T11:16:40,877 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:40,878 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f3bf8a292101485fab1c2353985ceaef is 50, key is test_row_0/C:col10/1732446999630/Put/seqid=0 2024-11-24T11:16:40,879 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/87eaf4e455c44e80ad512b44f2a115b8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/87eaf4e455c44e80ad512b44f2a115b8 2024-11-24T11:16:40,883 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into 87eaf4e455c44e80ad512b44f2a115b8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:40,883 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:40,883 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732447000847; duration=0sec 2024-11-24T11:16:40,883 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:40,883 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:40,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742335_1511 (size=13085) 2024-11-24T11:16:40,891 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/f3bf8a292101485fab1c2353985ceaef as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f3bf8a292101485fab1c2353985ceaef 2024-11-24T11:16:40,894 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into f3bf8a292101485fab1c2353985ceaef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:40,895 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:40,895 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732447000847; duration=0sec 2024-11-24T11:16:40,895 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:40,895 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T11:16:40,984 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:40,984 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-24T11:16:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T11:16:40,986 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:40,986 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:40,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T11:16:41,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T11:16:41,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:41,138 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:16:41,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:41,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:41,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/14e82d5fa53448f5b392cb5ff4675af3 is 50, key is test_row_0/A:col10/1732447000064/Put/seqid=0 2024-11-24T11:16:41,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742336_1512 (size=12301) 2024-11-24T11:16:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:41,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T11:16:41,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447061325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:41,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447061433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:41,550 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/14e82d5fa53448f5b392cb5ff4675af3 2024-11-24T11:16:41,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a5aec980fe6942c39520770dc6c567b3 is 50, key is test_row_0/B:col10/1732447000064/Put/seqid=0 2024-11-24T11:16:41,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742337_1513 (size=12301) 2024-11-24T11:16:41,563 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a5aec980fe6942c39520770dc6c567b3 2024-11-24T11:16:41,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/0196f90feb694fb19f8207d80d7f364a is 50, key is test_row_0/C:col10/1732447000064/Put/seqid=0 2024-11-24T11:16:41,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742338_1514 (size=12301) 2024-11-24T11:16:41,572 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/0196f90feb694fb19f8207d80d7f364a 2024-11-24T11:16:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/14e82d5fa53448f5b392cb5ff4675af3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/14e82d5fa53448f5b392cb5ff4675af3 2024-11-24T11:16:41,578 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/14e82d5fa53448f5b392cb5ff4675af3, entries=150, sequenceid=327, filesize=12.0 K 2024-11-24T11:16:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a5aec980fe6942c39520770dc6c567b3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a5aec980fe6942c39520770dc6c567b3 2024-11-24T11:16:41,581 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a5aec980fe6942c39520770dc6c567b3, entries=150, sequenceid=327, filesize=12.0 K 2024-11-24T11:16:41,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/0196f90feb694fb19f8207d80d7f364a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/0196f90feb694fb19f8207d80d7f364a 2024-11-24T11:16:41,584 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/0196f90feb694fb19f8207d80d7f364a, entries=150, sequenceid=327, filesize=12.0 K 2024-11-24T11:16:41,584 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 58548ec0d65011347d1619ef7b0b7484 in 446ms, sequenceid=327, compaction requested=false 2024-11-24T11:16:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:41,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-24T11:16:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-24T11:16:41,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-24T11:16:41,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 600 msec 2024-11-24T11:16:41,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 604 msec 2024-11-24T11:16:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T11:16:41,614 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:41,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-24T11:16:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T11:16:41,616 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:41,619 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:41,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:41,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:16:41,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:41,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:41,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:41,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:41,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:41,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:41,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/e98cac30b9154db38f6061e9ef95e1b6 is 50, key is test_row_0/A:col10/1732447001295/Put/seqid=0 2024-11-24T11:16:41,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742339_1515 (size=14741) 2024-11-24T11:16:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447061710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T11:16:41,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T11:16:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:41,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447061823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:41,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T11:16:41,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:41,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:41,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:41,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:41,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T11:16:42,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447062043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:42,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/e98cac30b9154db38f6061e9ef95e1b6 2024-11-24T11:16:42,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/8d89d0bdd9f8434bbe21d508300ff0c8 is 50, key is test_row_0/B:col10/1732447001295/Put/seqid=0 2024-11-24T11:16:42,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T11:16:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742340_1516 (size=12301) 2024-11-24T11:16:42,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T11:16:42,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T11:16:42,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447062364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:42,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T11:16:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:42,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:42,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/8d89d0bdd9f8434bbe21d508300ff0c8 2024-11-24T11:16:42,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/a0a62267bd7c490781429feec304f40a is 50, key is test_row_0/C:col10/1732447001295/Put/seqid=0 2024-11-24T11:16:42,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742341_1517 (size=12301) 2024-11-24T11:16:42,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/a0a62267bd7c490781429feec304f40a 2024-11-24T11:16:42,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/e98cac30b9154db38f6061e9ef95e1b6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e98cac30b9154db38f6061e9ef95e1b6 2024-11-24T11:16:42,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e98cac30b9154db38f6061e9ef95e1b6, entries=200, sequenceid=346, filesize=14.4 K 2024-11-24T11:16:42,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/8d89d0bdd9f8434bbe21d508300ff0c8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/8d89d0bdd9f8434bbe21d508300ff0c8 2024-11-24T11:16:42,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/8d89d0bdd9f8434bbe21d508300ff0c8, entries=150, sequenceid=346, filesize=12.0 K 2024-11-24T11:16:42,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/a0a62267bd7c490781429feec304f40a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/a0a62267bd7c490781429feec304f40a 2024-11-24T11:16:42,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/a0a62267bd7c490781429feec304f40a, entries=150, sequenceid=346, filesize=12.0 K 2024-11-24T11:16:42,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 58548ec0d65011347d1619ef7b0b7484 in 860ms, sequenceid=346, compaction requested=true 2024-11-24T11:16:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:42,511 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:42,511 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:42,512 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:42,512 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:42,512 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:42,512 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,512 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:42,512 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,512 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/87eaf4e455c44e80ad512b44f2a115b8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/14e82d5fa53448f5b392cb5ff4675af3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e98cac30b9154db38f6061e9ef95e1b6] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=39.2 K 2024-11-24T11:16:42,512 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/2720635d50fc4384801c0b69676fd2d9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a5aec980fe6942c39520770dc6c567b3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/8d89d0bdd9f8434bbe21d508300ff0c8] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.8 K 2024-11-24T11:16:42,512 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87eaf4e455c44e80ad512b44f2a115b8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732446999630 2024-11-24T11:16:42,512 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 2720635d50fc4384801c0b69676fd2d9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732446999630 2024-11-24T11:16:42,513 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e82d5fa53448f5b392cb5ff4675af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732447000057 2024-11-24T11:16:42,513 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a5aec980fe6942c39520770dc6c567b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732447000057 2024-11-24T11:16:42,513 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e98cac30b9154db38f6061e9ef95e1b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732447001273 2024-11-24T11:16:42,513 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d89d0bdd9f8434bbe21d508300ff0c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732447001295 2024-11-24T11:16:42,520 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:42,521 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/962476f957ef45a88dede6349bdb4d0d is 50, key is test_row_0/B:col10/1732447001295/Put/seqid=0 2024-11-24T11:16:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742342_1518 (size=13187) 2024-11-24T11:16:42,527 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:42,527 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ff21b12fd7fe487c9d3dd6d8b33a87ed is 50, key is test_row_0/A:col10/1732447001295/Put/seqid=0 2024-11-24T11:16:42,532 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/962476f957ef45a88dede6349bdb4d0d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/962476f957ef45a88dede6349bdb4d0d 2024-11-24T11:16:42,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742343_1519 (size=13187) 2024-11-24T11:16:42,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T11:16:42,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,535 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T11:16:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:42,536 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into 962476f957ef45a88dede6349bdb4d0d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:42,536 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:42,536 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732447002511; duration=0sec 2024-11-24T11:16:42,536 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:42,536 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:42,537 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:42,546 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:42,546 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:42,546 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:42,546 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f3bf8a292101485fab1c2353985ceaef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/0196f90feb694fb19f8207d80d7f364a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/a0a62267bd7c490781429feec304f40a] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.8 K 2024-11-24T11:16:42,546 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bf8a292101485fab1c2353985ceaef, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732446999630 2024-11-24T11:16:42,547 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 0196f90feb694fb19f8207d80d7f364a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732447000057 2024-11-24T11:16:42,547 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a0a62267bd7c490781429feec304f40a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732447001295 2024-11-24T11:16:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/aa2491eca4154a25af1f12de544c6358 is 50, key is test_row_0/A:col10/1732447001692/Put/seqid=0 2024-11-24T11:16:42,558 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:42,558 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/b7ed028b5680418295d4e034348efe3e is 50, key is test_row_0/C:col10/1732447001295/Put/seqid=0 2024-11-24T11:16:42,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742344_1520 (size=12301) 2024-11-24T11:16:42,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742345_1521 (size=13187) 2024-11-24T11:16:42,570 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/aa2491eca4154a25af1f12de544c6358 2024-11-24T11:16:42,575 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/b7ed028b5680418295d4e034348efe3e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b7ed028b5680418295d4e034348efe3e 2024-11-24T11:16:42,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/69982f92293445eba505dca4901a36f8 is 50, key is test_row_0/B:col10/1732447001692/Put/seqid=0 2024-11-24T11:16:42,587 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into b7ed028b5680418295d4e034348efe3e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:42,587 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:42,587 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732447002511; duration=0sec 2024-11-24T11:16:42,587 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:42,587 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:42,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742346_1522 (size=12301) 2024-11-24T11:16:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T11:16:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:42,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:42,939 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/ff21b12fd7fe487c9d3dd6d8b33a87ed as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ff21b12fd7fe487c9d3dd6d8b33a87ed 2024-11-24T11:16:42,952 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into ff21b12fd7fe487c9d3dd6d8b33a87ed(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:42,952 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:42,952 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732447002511; duration=0sec 2024-11-24T11:16:42,953 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:42,953 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:42,993 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/69982f92293445eba505dca4901a36f8 2024-11-24T11:16:42,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447062992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:42,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/3d6634390cb7455da20801f0a09af85d is 50, key is test_row_0/C:col10/1732447001692/Put/seqid=0 2024-11-24T11:16:43,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742347_1523 (size=12301) 2024-11-24T11:16:43,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447063103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447063314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:43,405 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/3d6634390cb7455da20801f0a09af85d 2024-11-24T11:16:43,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/aa2491eca4154a25af1f12de544c6358 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa2491eca4154a25af1f12de544c6358 2024-11-24T11:16:43,411 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa2491eca4154a25af1f12de544c6358, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T11:16:43,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/69982f92293445eba505dca4901a36f8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/69982f92293445eba505dca4901a36f8 2024-11-24T11:16:43,414 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/69982f92293445eba505dca4901a36f8, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T11:16:43,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/3d6634390cb7455da20801f0a09af85d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/3d6634390cb7455da20801f0a09af85d 2024-11-24T11:16:43,418 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/3d6634390cb7455da20801f0a09af85d, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T11:16:43,419 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 58548ec0d65011347d1619ef7b0b7484 in 884ms, sequenceid=359, compaction requested=false 2024-11-24T11:16:43,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:43,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:43,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-24T11:16:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-24T11:16:43,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-24T11:16:43,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-11-24T11:16:43,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.8080 sec 2024-11-24T11:16:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:43,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T11:16:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:43,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/dca6eb1f29a3469ea7ef535c47346805 is 50, key is test_row_0/A:col10/1732447002990/Put/seqid=0 2024-11-24T11:16:43,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742348_1524 (size=14741) 2024-11-24T11:16:43,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447063694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T11:16:43,763 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:43,764 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-24T11:16:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T11:16:43,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:16:43,766 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:16:43,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:16:43,773 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-24T11:16:43,773 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:43,773 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,773 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:43,777 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-24T11:16:43,777 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:43,777 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,777 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:43,782 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-11-24T11:16:43,782 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:43,782 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,782 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:43,783 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-24T11:16:43,783 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:43,783 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,783 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:43,786 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-24T11:16:43,786 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:43,786 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,786 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:43,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447063803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T11:16:43,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:43,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447064014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:44,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/dca6eb1f29a3469ea7ef535c47346805 2024-11-24T11:16:44,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/510bf79ae64047b6a1ce583c185d14a5 is 50, key is test_row_0/B:col10/1732447002990/Put/seqid=0 2024-11-24T11:16:44,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742349_1525 (size=12301) 2024-11-24T11:16:44,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:44,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:44,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T11:16:44,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447064325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:44,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:44,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:44,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T11:16:44,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/510bf79ae64047b6a1ce583c185d14a5 2024-11-24T11:16:44,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/505205c052394c73a5c9e02bf34971fc is 50, key is test_row_0/C:col10/1732447002990/Put/seqid=0 2024-11-24T11:16:44,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742350_1526 (size=12301) 2024-11-24T11:16:44,520 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T11:16:44,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:44,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:44,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44018 deadline: 1732447064755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:44,757 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:44,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44006 deadline: 1732447064794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:44,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:44,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44046 deadline: 1732447064795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43996 deadline: 1732447064795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:44,796 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18338 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:44,796 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18329 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:44,796 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in 58548ec0d65011347d1619ef7b0b7484 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18339 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:16:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:44,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:16:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44032 deadline: 1732447064834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 , the old value is region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:16:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., hostname=f5f73e76a839,45939,1732446856176, seqNum=2 because the exception is null or not the one we care about 2024-11-24T11:16:44,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:16:44,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/505205c052394c73a5c9e02bf34971fc 2024-11-24T11:16:44,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/dca6eb1f29a3469ea7ef535c47346805 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/dca6eb1f29a3469ea7ef535c47346805 2024-11-24T11:16:44,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/dca6eb1f29a3469ea7ef535c47346805, entries=200, sequenceid=382, filesize=14.4 K 2024-11-24T11:16:44,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/510bf79ae64047b6a1ce583c185d14a5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/510bf79ae64047b6a1ce583c185d14a5 2024-11-24T11:16:44,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/510bf79ae64047b6a1ce583c185d14a5, entries=150, sequenceid=382, filesize=12.0 K 2024-11-24T11:16:44,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/505205c052394c73a5c9e02bf34971fc as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/505205c052394c73a5c9e02bf34971fc 2024-11-24T11:16:44,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/505205c052394c73a5c9e02bf34971fc, entries=150, sequenceid=382, filesize=12.0 K 2024-11-24T11:16:44,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 58548ec0d65011347d1619ef7b0b7484 in 1250ms, sequenceid=382, compaction requested=true 2024-11-24T11:16:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:16:44,877 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:16:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58548ec0d65011347d1619ef7b0b7484:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:16:44,877 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:44,878 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:44,878 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:44,878 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/A is initiating minor compaction (all files) 2024-11-24T11:16:44,878 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/B is initiating minor compaction (all files) 2024-11-24T11:16:44,878 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/A in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,878 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/B in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,879 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/962476f957ef45a88dede6349bdb4d0d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/69982f92293445eba505dca4901a36f8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/510bf79ae64047b6a1ce583c185d14a5] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.9 K 2024-11-24T11:16:44,879 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ff21b12fd7fe487c9d3dd6d8b33a87ed, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa2491eca4154a25af1f12de544c6358, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/dca6eb1f29a3469ea7ef535c47346805] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=39.3 K 2024-11-24T11:16:44,879 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 962476f957ef45a88dede6349bdb4d0d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732447001295 2024-11-24T11:16:44,879 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff21b12fd7fe487c9d3dd6d8b33a87ed, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732447001295 2024-11-24T11:16:44,879 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 69982f92293445eba505dca4901a36f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732447001669 2024-11-24T11:16:44,879 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa2491eca4154a25af1f12de544c6358, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732447001669 2024-11-24T11:16:44,880 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 510bf79ae64047b6a1ce583c185d14a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732447002920 2024-11-24T11:16:44,880 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca6eb1f29a3469ea7ef535c47346805, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732447002920 2024-11-24T11:16:44,889 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#B#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:44,889 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#A#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:44,890 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4b5b436e549e44fb86ed96488a3f5dbe is 50, key is test_row_0/B:col10/1732447002990/Put/seqid=0 2024-11-24T11:16:44,890 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/afd3c7b6d7e04ca0a21dacfe070d5733 is 50, key is test_row_0/A:col10/1732447002990/Put/seqid=0 2024-11-24T11:16:44,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742351_1527 (size=13289) 2024-11-24T11:16:44,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742352_1528 (size=13289) 2024-11-24T11:16:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T11:16:44,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T11:16:44,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:44,996 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T11:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/9fbf0551d89e4523a9024c4b5c9bb350 is 50, key is test_row_0/A:col10/1732447003693/Put/seqid=0 2024-11-24T11:16:45,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742353_1529 (size=12301) 2024-11-24T11:16:45,296 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4b5b436e549e44fb86ed96488a3f5dbe as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4b5b436e549e44fb86ed96488a3f5dbe 2024-11-24T11:16:45,296 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/afd3c7b6d7e04ca0a21dacfe070d5733 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/afd3c7b6d7e04ca0a21dacfe070d5733 2024-11-24T11:16:45,299 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/B of 58548ec0d65011347d1619ef7b0b7484 into 4b5b436e549e44fb86ed96488a3f5dbe(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:45,299 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/A of 58548ec0d65011347d1619ef7b0b7484 into afd3c7b6d7e04ca0a21dacfe070d5733(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:45,299 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:45,299 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:45,299 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/A, priority=13, startTime=1732447004877; duration=0sec 2024-11-24T11:16:45,299 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/B, priority=13, startTime=1732447004877; duration=0sec 2024-11-24T11:16:45,299 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:16:45,299 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:B 2024-11-24T11:16:45,299 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:45,299 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:A 2024-11-24T11:16:45,299 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:16:45,300 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:16:45,300 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): 58548ec0d65011347d1619ef7b0b7484/C is initiating minor compaction (all files) 2024-11-24T11:16:45,300 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58548ec0d65011347d1619ef7b0b7484/C in TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:45,300 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b7ed028b5680418295d4e034348efe3e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/3d6634390cb7455da20801f0a09af85d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/505205c052394c73a5c9e02bf34971fc] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp, totalSize=36.9 K 2024-11-24T11:16:45,300 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ed028b5680418295d4e034348efe3e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732447001295 2024-11-24T11:16:45,300 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d6634390cb7455da20801f0a09af85d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732447001669 2024-11-24T11:16:45,301 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 505205c052394c73a5c9e02bf34971fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732447002920 2024-11-24T11:16:45,305 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58548ec0d65011347d1619ef7b0b7484#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:16:45,306 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/6b4ae1ee85d04950bfda18fb1b9c04ac is 50, key is test_row_0/C:col10/1732447002990/Put/seqid=0 2024-11-24T11:16:45,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742354_1530 (size=13289) 2024-11-24T11:16:45,408 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/9fbf0551d89e4523a9024c4b5c9bb350 2024-11-24T11:16:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4e563d15142545a7979a76e3ede178fa is 50, key is test_row_0/B:col10/1732447003693/Put/seqid=0 2024-11-24T11:16:45,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742355_1531 (size=12301) 2024-11-24T11:16:45,712 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/6b4ae1ee85d04950bfda18fb1b9c04ac as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/6b4ae1ee85d04950bfda18fb1b9c04ac 2024-11-24T11:16:45,717 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58548ec0d65011347d1619ef7b0b7484/C of 58548ec0d65011347d1619ef7b0b7484 into 6b4ae1ee85d04950bfda18fb1b9c04ac(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:16:45,717 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:45,717 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., storeName=58548ec0d65011347d1619ef7b0b7484/C, priority=13, startTime=1732447004877; duration=0sec 2024-11-24T11:16:45,717 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:16:45,717 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58548ec0d65011347d1619ef7b0b7484:C 2024-11-24T11:16:45,826 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4e563d15142545a7979a76e3ede178fa 2024-11-24T11:16:45,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/e1b2aacf39b6400cb57c5b1aa9202276 is 50, key is test_row_0/C:col10/1732447003693/Put/seqid=0 2024-11-24T11:16:45,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742356_1532 (size=12301) 2024-11-24T11:16:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:45,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. as already flushing 2024-11-24T11:16:45,845 INFO [Thread-1984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1984. 2024-11-24T11:16:45,845 DEBUG [Thread-1984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:45,845 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:45,845 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:45,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T11:16:46,239 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/e1b2aacf39b6400cb57c5b1aa9202276 2024-11-24T11:16:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/9fbf0551d89e4523a9024c4b5c9bb350 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9fbf0551d89e4523a9024c4b5c9bb350 2024-11-24T11:16:46,246 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9fbf0551d89e4523a9024c4b5c9bb350, entries=150, sequenceid=394, filesize=12.0 K 2024-11-24T11:16:46,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/4e563d15142545a7979a76e3ede178fa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4e563d15142545a7979a76e3ede178fa 2024-11-24T11:16:46,249 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4e563d15142545a7979a76e3ede178fa, entries=150, sequenceid=394, filesize=12.0 K 2024-11-24T11:16:46,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/e1b2aacf39b6400cb57c5b1aa9202276 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e1b2aacf39b6400cb57c5b1aa9202276 2024-11-24T11:16:46,252 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e1b2aacf39b6400cb57c5b1aa9202276, entries=150, sequenceid=394, filesize=12.0 K 2024-11-24T11:16:46,253 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for 58548ec0d65011347d1619ef7b0b7484 in 1257ms, sequenceid=394, compaction requested=false 2024-11-24T11:16:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 58548ec0d65011347d1619ef7b0b7484: 2024-11-24T11:16:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-24T11:16:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-24T11:16:46,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-24T11:16:46,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4880 sec 2024-11-24T11:16:46,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.4920 sec 2024-11-24T11:16:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T11:16:47,945 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:54,815 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-24T11:16:54,816 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:54,816 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,816 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:54,835 INFO [Thread-1978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1978. 2024-11-24T11:16:54,835 DEBUG [Thread-1978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:54,835 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,835 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:54,845 INFO [Thread-1976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1976. 2024-11-24T11:16:54,845 DEBUG [Thread-1976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:54,845 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,845 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:54,855 INFO [Thread-1980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1980. 2024-11-24T11:16:54,855 DEBUG [Thread-1980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:16:54,855 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,855 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:54,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T11:16:54,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-24T11:16:54,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-24T11:16:54,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-24T11:16:54,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-24T11:16:54,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-11-24T11:16:54,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3162 rows 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1062 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3186 rows 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1055 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3162 rows 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1052 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3155 rows 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1057 2024-11-24T11:16:54,856 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3168 rows 2024-11-24T11:16:54,856 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:16:54,856 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:16:54,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:54,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:16:54,856 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-24T11:16:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T11:16:54,859 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732447014859"}]},"ts":"1732447014859"} 2024-11-24T11:16:54,860 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T11:16:54,860 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T11:16:54,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:16:54,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, UNASSIGN}] 2024-11-24T11:16:54,862 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, UNASSIGN 2024-11-24T11:16:54,863 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=58548ec0d65011347d1619ef7b0b7484, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:54,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, UNASSIGN because future has completed 2024-11-24T11:16:54,864 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:16:54,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:16:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T11:16:55,016 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:55,016 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:16:55,016 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 58548ec0d65011347d1619ef7b0b7484, disabling compactions & flushes 2024-11-24T11:16:55,017 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. after waiting 0 ms 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:55,017 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 58548ec0d65011347d1619ef7b0b7484 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=A 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=B 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58548ec0d65011347d1619ef7b0b7484, store=C 2024-11-24T11:16:55,017 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:16:55,023 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/e1439df9e7534b9d84cee0c98cf2364e is 50, key is test_row_0/A:col10/1732447014833/Put/seqid=0 2024-11-24T11:16:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742357_1533 (size=12301) 2024-11-24T11:16:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T11:16:55,431 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/e1439df9e7534b9d84cee0c98cf2364e 2024-11-24T11:16:55,445 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a40c5189aeb34cbd8ca1ab90633f97f8 is 50, key is test_row_0/B:col10/1732447014833/Put/seqid=0 2024-11-24T11:16:55,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742358_1534 (size=12301) 2024-11-24T11:16:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T11:16:55,850 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a40c5189aeb34cbd8ca1ab90633f97f8 2024-11-24T11:16:55,865 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/55f2e79c04034579b752e0ae56653331 is 50, key is test_row_0/C:col10/1732447014833/Put/seqid=0 2024-11-24T11:16:55,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742359_1535 (size=12301) 2024-11-24T11:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T11:16:56,270 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/55f2e79c04034579b752e0ae56653331 2024-11-24T11:16:56,279 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/A/e1439df9e7534b9d84cee0c98cf2364e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e1439df9e7534b9d84cee0c98cf2364e 2024-11-24T11:16:56,283 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e1439df9e7534b9d84cee0c98cf2364e, entries=150, sequenceid=405, filesize=12.0 K 2024-11-24T11:16:56,284 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/B/a40c5189aeb34cbd8ca1ab90633f97f8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a40c5189aeb34cbd8ca1ab90633f97f8 2024-11-24T11:16:56,288 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a40c5189aeb34cbd8ca1ab90633f97f8, entries=150, sequenceid=405, filesize=12.0 K 2024-11-24T11:16:56,289 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/.tmp/C/55f2e79c04034579b752e0ae56653331 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/55f2e79c04034579b752e0ae56653331 2024-11-24T11:16:56,293 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/55f2e79c04034579b752e0ae56653331, entries=150, sequenceid=405, filesize=12.0 K 2024-11-24T11:16:56,294 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 58548ec0d65011347d1619ef7b0b7484 in 1277ms, sequenceid=405, compaction requested=true 2024-11-24T11:16:56,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/658302356ecb4373a0689ac4dd2bd0b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b40bc000085a449b9bfd9177a95a3d6d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/83a4810f0dc64d8985a87923e4e8b6c1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/74911627db3d4b04a36d8041de048881, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/26abf4977cac41139b4dd5dc15a8be1c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/f12be9bde8374fdabe656a1c43b0dedd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/bf890057972b4f6ab4f435c3b4507f4c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/306921faa6c7434099faf87982455696, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/859460d62a4c416bacf65e2132223e5f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b139015db12a4b708448a2859b9042bd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9e57a493890a472e94db83cc60a7d038, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/55a4d841b0e145f78fe033dcb70c1a9a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/12eea9b3f79642d18ade35f1d45fb5c4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa9e8bf277804f3d99b48e6856ddebb1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/157adc329062435099668847a9cb6e0a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ac071d7cd1be44ac9434c19fe13eac5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/47a83467cc334839ad47c883aa978bd7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/44f56587ddb4429c8963d4ce7da6ff65, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ccec2d8d43244914a1356f999bcbc550, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/45c7e32ac02142beb4b976024c5cadd9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/be6350de4bf44a72bd73a8aa5db55d79, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b04d9791ee3c489991035122ff05f81c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/87eaf4e455c44e80ad512b44f2a115b8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/14e82d5fa53448f5b392cb5ff4675af3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e98cac30b9154db38f6061e9ef95e1b6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ff21b12fd7fe487c9d3dd6d8b33a87ed, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa2491eca4154a25af1f12de544c6358, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/dca6eb1f29a3469ea7ef535c47346805] to archive 2024-11-24T11:16:56,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:16:56,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/658302356ecb4373a0689ac4dd2bd0b6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/658302356ecb4373a0689ac4dd2bd0b6 2024-11-24T11:16:56,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b40bc000085a449b9bfd9177a95a3d6d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b40bc000085a449b9bfd9177a95a3d6d 2024-11-24T11:16:56,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/83a4810f0dc64d8985a87923e4e8b6c1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/83a4810f0dc64d8985a87923e4e8b6c1 2024-11-24T11:16:56,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/74911627db3d4b04a36d8041de048881 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/74911627db3d4b04a36d8041de048881 2024-11-24T11:16:56,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/26abf4977cac41139b4dd5dc15a8be1c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/26abf4977cac41139b4dd5dc15a8be1c 2024-11-24T11:16:56,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/4cdaf3811e1b4ec5a18dcc142a6ed6ca 2024-11-24T11:16:56,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/f12be9bde8374fdabe656a1c43b0dedd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/f12be9bde8374fdabe656a1c43b0dedd 2024-11-24T11:16:56,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/bf890057972b4f6ab4f435c3b4507f4c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/bf890057972b4f6ab4f435c3b4507f4c 2024-11-24T11:16:56,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/306921faa6c7434099faf87982455696 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/306921faa6c7434099faf87982455696 2024-11-24T11:16:56,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/859460d62a4c416bacf65e2132223e5f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/859460d62a4c416bacf65e2132223e5f 2024-11-24T11:16:56,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b139015db12a4b708448a2859b9042bd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b139015db12a4b708448a2859b9042bd 2024-11-24T11:16:56,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9e57a493890a472e94db83cc60a7d038 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9e57a493890a472e94db83cc60a7d038 2024-11-24T11:16:56,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/55a4d841b0e145f78fe033dcb70c1a9a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/55a4d841b0e145f78fe033dcb70c1a9a 2024-11-24T11:16:56,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/12eea9b3f79642d18ade35f1d45fb5c4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/12eea9b3f79642d18ade35f1d45fb5c4 2024-11-24T11:16:56,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa9e8bf277804f3d99b48e6856ddebb1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa9e8bf277804f3d99b48e6856ddebb1 2024-11-24T11:16:56,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/157adc329062435099668847a9cb6e0a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/157adc329062435099668847a9cb6e0a 2024-11-24T11:16:56,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ac071d7cd1be44ac9434c19fe13eac5b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ac071d7cd1be44ac9434c19fe13eac5b 2024-11-24T11:16:56,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/47a83467cc334839ad47c883aa978bd7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/47a83467cc334839ad47c883aa978bd7 2024-11-24T11:16:56,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/44f56587ddb4429c8963d4ce7da6ff65 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/44f56587ddb4429c8963d4ce7da6ff65 2024-11-24T11:16:56,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ccec2d8d43244914a1356f999bcbc550 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ccec2d8d43244914a1356f999bcbc550 2024-11-24T11:16:56,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/45c7e32ac02142beb4b976024c5cadd9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/45c7e32ac02142beb4b976024c5cadd9 2024-11-24T11:16:56,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/be6350de4bf44a72bd73a8aa5db55d79 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/be6350de4bf44a72bd73a8aa5db55d79 2024-11-24T11:16:56,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b04d9791ee3c489991035122ff05f81c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/b04d9791ee3c489991035122ff05f81c 2024-11-24T11:16:56,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/87eaf4e455c44e80ad512b44f2a115b8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/87eaf4e455c44e80ad512b44f2a115b8 2024-11-24T11:16:56,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/14e82d5fa53448f5b392cb5ff4675af3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/14e82d5fa53448f5b392cb5ff4675af3 2024-11-24T11:16:56,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e98cac30b9154db38f6061e9ef95e1b6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e98cac30b9154db38f6061e9ef95e1b6 2024-11-24T11:16:56,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ff21b12fd7fe487c9d3dd6d8b33a87ed to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/ff21b12fd7fe487c9d3dd6d8b33a87ed 2024-11-24T11:16:56,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa2491eca4154a25af1f12de544c6358 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/aa2491eca4154a25af1f12de544c6358 2024-11-24T11:16:56,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/dca6eb1f29a3469ea7ef535c47346805 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/dca6eb1f29a3469ea7ef535c47346805 2024-11-24T11:16:56,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/254ac1b052cc4423bd2aa4cf1524c762, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/cc49023b62ec4c31a5c1115fa03e2063, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/ce7222c5eb7c4770b7b4536973f8bd8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7ca21586489c4b4b8169569d6ff8d2e2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b02ba69ff686444795703f449d2331a6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/5324719602e74d2491aa54c9c0eb4d21, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7ff6d871c8a4aff9b2f2ce648225520, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a2bd07afdce045d6b66255074e1180bd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/07f033d732ad46759ecc2e0560b0af6f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/9b6c2f1211ef4ab0a7053a0f11105183, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/72ef2c8a20a340ec850f1d24afdcdf9b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7df75acc55a1402dafcb6adc3222684d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b4d610e3e0c0448e9313e7b139edbfd6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b2367044622d44c4acfcc5de0bf34eef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/163d8a4e97b9425c96474898fd39b8ab, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f69a81e9bfe7450fbfd199ec0552c8c0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/37cf9c92a5774e009f3e40a54916d521, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a3950ecea219461c829728d9dc5e5b8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/53c8446971a24650a0cd01ef6636df21, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/fe423218ecf84bf18cb25ba545db23f2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4c30b19a15e24d3a80fe4a820433ed0d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7041753a67e4975b03ae3f5aa8aacca, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/2720635d50fc4384801c0b69676fd2d9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/784aa20baacb4ce6a05a48896df92a2c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a5aec980fe6942c39520770dc6c567b3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/962476f957ef45a88dede6349bdb4d0d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/8d89d0bdd9f8434bbe21d508300ff0c8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/69982f92293445eba505dca4901a36f8, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/510bf79ae64047b6a1ce583c185d14a5] to archive 2024-11-24T11:16:56,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:16:56,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/254ac1b052cc4423bd2aa4cf1524c762 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/254ac1b052cc4423bd2aa4cf1524c762 2024-11-24T11:16:56,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/cc49023b62ec4c31a5c1115fa03e2063 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/cc49023b62ec4c31a5c1115fa03e2063 2024-11-24T11:16:56,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/ce7222c5eb7c4770b7b4536973f8bd8d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/ce7222c5eb7c4770b7b4536973f8bd8d 2024-11-24T11:16:56,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7ca21586489c4b4b8169569d6ff8d2e2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7ca21586489c4b4b8169569d6ff8d2e2 2024-11-24T11:16:56,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b02ba69ff686444795703f449d2331a6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b02ba69ff686444795703f449d2331a6 2024-11-24T11:16:56,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/5324719602e74d2491aa54c9c0eb4d21 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/5324719602e74d2491aa54c9c0eb4d21 2024-11-24T11:16:56,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7ff6d871c8a4aff9b2f2ce648225520 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7ff6d871c8a4aff9b2f2ce648225520 2024-11-24T11:16:56,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a2bd07afdce045d6b66255074e1180bd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a2bd07afdce045d6b66255074e1180bd 2024-11-24T11:16:56,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/07f033d732ad46759ecc2e0560b0af6f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/07f033d732ad46759ecc2e0560b0af6f 2024-11-24T11:16:56,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/9b6c2f1211ef4ab0a7053a0f11105183 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/9b6c2f1211ef4ab0a7053a0f11105183 2024-11-24T11:16:56,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/72ef2c8a20a340ec850f1d24afdcdf9b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/72ef2c8a20a340ec850f1d24afdcdf9b 2024-11-24T11:16:56,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7df75acc55a1402dafcb6adc3222684d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/7df75acc55a1402dafcb6adc3222684d 2024-11-24T11:16:56,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b4d610e3e0c0448e9313e7b139edbfd6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b4d610e3e0c0448e9313e7b139edbfd6 2024-11-24T11:16:56,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b2367044622d44c4acfcc5de0bf34eef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/b2367044622d44c4acfcc5de0bf34eef 2024-11-24T11:16:56,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/163d8a4e97b9425c96474898fd39b8ab to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/163d8a4e97b9425c96474898fd39b8ab 2024-11-24T11:16:56,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f69a81e9bfe7450fbfd199ec0552c8c0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f69a81e9bfe7450fbfd199ec0552c8c0 2024-11-24T11:16:56,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/37cf9c92a5774e009f3e40a54916d521 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/37cf9c92a5774e009f3e40a54916d521 2024-11-24T11:16:56,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a3950ecea219461c829728d9dc5e5b8d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a3950ecea219461c829728d9dc5e5b8d 2024-11-24T11:16:56,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/53c8446971a24650a0cd01ef6636df21 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/53c8446971a24650a0cd01ef6636df21 2024-11-24T11:16:56,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/fe423218ecf84bf18cb25ba545db23f2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/fe423218ecf84bf18cb25ba545db23f2 2024-11-24T11:16:56,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4c30b19a15e24d3a80fe4a820433ed0d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4c30b19a15e24d3a80fe4a820433ed0d 2024-11-24T11:16:56,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7041753a67e4975b03ae3f5aa8aacca to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/f7041753a67e4975b03ae3f5aa8aacca 2024-11-24T11:16:56,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/2720635d50fc4384801c0b69676fd2d9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/2720635d50fc4384801c0b69676fd2d9 2024-11-24T11:16:56,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/784aa20baacb4ce6a05a48896df92a2c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/784aa20baacb4ce6a05a48896df92a2c 2024-11-24T11:16:56,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a5aec980fe6942c39520770dc6c567b3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a5aec980fe6942c39520770dc6c567b3 2024-11-24T11:16:56,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/962476f957ef45a88dede6349bdb4d0d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/962476f957ef45a88dede6349bdb4d0d 2024-11-24T11:16:56,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/8d89d0bdd9f8434bbe21d508300ff0c8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/8d89d0bdd9f8434bbe21d508300ff0c8 2024-11-24T11:16:56,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/69982f92293445eba505dca4901a36f8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/69982f92293445eba505dca4901a36f8 2024-11-24T11:16:56,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/510bf79ae64047b6a1ce583c185d14a5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/510bf79ae64047b6a1ce583c185d14a5 2024-11-24T11:16:56,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/377a5374a6c948c9ae3c4bb5df2950b0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07107716e70f4a2b95aa74735561b3f5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/efd3b7211a20455d912862911e6a2345, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/16de1b8a1a5e4d808b83bb3d5482a492, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/c1ea4b4dc5c14874ad14eb1524af10c6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/8b5b6478346e4ce08210624e33a70221, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/5caa9cbaee314b848df56d8bc65a8d1f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/fe3bb2079dfb44ac8531cfc3f664b2ad, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/7608ea678bd6486785c4bb6225d98c4a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d27a44055a734ce39f1f4f345d72fa8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/4247a007f67d4d4d86f69029e086a672, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0fcdcf757074425bd93e5d27714ceee, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/62cbe063e3184fadb6760bfd8eccb899, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/551cfe742328447f949ac5976747089e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/1e1766d7988447908f8e312095efeee9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b29f4e956a374bf985fc67a88ece2cb5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e0acc7c80a7a4fe999a647fc7368efa6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0ec4d2a1b834bd1a3340b7a51962503, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/cad80cbc31ad4f319653a9504f692c90, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/9275a6f7eb464360bfe0db18bd3b8411, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d4a2a96402fc47e38a2ef7ab96a39b34, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07649c572ef34c0db8ef57bde7167d37, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f3bf8a292101485fab1c2353985ceaef, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/32a9b125cf674b1fb848e6d151ed5a4d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/0196f90feb694fb19f8207d80d7f364a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b7ed028b5680418295d4e034348efe3e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/a0a62267bd7c490781429feec304f40a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/3d6634390cb7455da20801f0a09af85d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/505205c052394c73a5c9e02bf34971fc] to archive 2024-11-24T11:16:56,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:16:56,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/377a5374a6c948c9ae3c4bb5df2950b0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/377a5374a6c948c9ae3c4bb5df2950b0 2024-11-24T11:16:56,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07107716e70f4a2b95aa74735561b3f5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07107716e70f4a2b95aa74735561b3f5 2024-11-24T11:16:56,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/efd3b7211a20455d912862911e6a2345 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/efd3b7211a20455d912862911e6a2345 2024-11-24T11:16:56,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/16de1b8a1a5e4d808b83bb3d5482a492 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/16de1b8a1a5e4d808b83bb3d5482a492 2024-11-24T11:16:56,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/c1ea4b4dc5c14874ad14eb1524af10c6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/c1ea4b4dc5c14874ad14eb1524af10c6 2024-11-24T11:16:56,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/8b5b6478346e4ce08210624e33a70221 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/8b5b6478346e4ce08210624e33a70221 2024-11-24T11:16:56,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/5caa9cbaee314b848df56d8bc65a8d1f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/5caa9cbaee314b848df56d8bc65a8d1f 2024-11-24T11:16:56,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/fe3bb2079dfb44ac8531cfc3f664b2ad to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/fe3bb2079dfb44ac8531cfc3f664b2ad 2024-11-24T11:16:56,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/7608ea678bd6486785c4bb6225d98c4a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/7608ea678bd6486785c4bb6225d98c4a 2024-11-24T11:16:56,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d27a44055a734ce39f1f4f345d72fa8d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d27a44055a734ce39f1f4f345d72fa8d 2024-11-24T11:16:56,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/4247a007f67d4d4d86f69029e086a672 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/4247a007f67d4d4d86f69029e086a672 2024-11-24T11:16:56,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0fcdcf757074425bd93e5d27714ceee to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0fcdcf757074425bd93e5d27714ceee 2024-11-24T11:16:56,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/62cbe063e3184fadb6760bfd8eccb899 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/62cbe063e3184fadb6760bfd8eccb899 2024-11-24T11:16:56,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/551cfe742328447f949ac5976747089e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/551cfe742328447f949ac5976747089e 2024-11-24T11:16:56,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/1e1766d7988447908f8e312095efeee9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/1e1766d7988447908f8e312095efeee9 2024-11-24T11:16:56,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b29f4e956a374bf985fc67a88ece2cb5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b29f4e956a374bf985fc67a88ece2cb5 2024-11-24T11:16:56,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e0acc7c80a7a4fe999a647fc7368efa6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e0acc7c80a7a4fe999a647fc7368efa6 2024-11-24T11:16:56,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0ec4d2a1b834bd1a3340b7a51962503 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f0ec4d2a1b834bd1a3340b7a51962503 2024-11-24T11:16:56,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/cad80cbc31ad4f319653a9504f692c90 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/cad80cbc31ad4f319653a9504f692c90 2024-11-24T11:16:56,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/9275a6f7eb464360bfe0db18bd3b8411 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/9275a6f7eb464360bfe0db18bd3b8411 2024-11-24T11:16:56,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d4a2a96402fc47e38a2ef7ab96a39b34 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/d4a2a96402fc47e38a2ef7ab96a39b34 2024-11-24T11:16:56,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07649c572ef34c0db8ef57bde7167d37 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/07649c572ef34c0db8ef57bde7167d37 2024-11-24T11:16:56,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f3bf8a292101485fab1c2353985ceaef to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/f3bf8a292101485fab1c2353985ceaef 2024-11-24T11:16:56,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/32a9b125cf674b1fb848e6d151ed5a4d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/32a9b125cf674b1fb848e6d151ed5a4d 2024-11-24T11:16:56,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/0196f90feb694fb19f8207d80d7f364a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/0196f90feb694fb19f8207d80d7f364a 2024-11-24T11:16:56,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b7ed028b5680418295d4e034348efe3e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/b7ed028b5680418295d4e034348efe3e 2024-11-24T11:16:56,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/a0a62267bd7c490781429feec304f40a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/a0a62267bd7c490781429feec304f40a 2024-11-24T11:16:56,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/3d6634390cb7455da20801f0a09af85d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/3d6634390cb7455da20801f0a09af85d 2024-11-24T11:16:56,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/505205c052394c73a5c9e02bf34971fc to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/505205c052394c73a5c9e02bf34971fc 2024-11-24T11:16:56,389 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/recovered.edits/408.seqid, newMaxSeqId=408, maxSeqId=1 2024-11-24T11:16:56,390 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. 2024-11-24T11:16:56,390 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 58548ec0d65011347d1619ef7b0b7484: Waiting for close lock at 1732447015016Running coprocessor pre-close hooks at 1732447015016Disabling compacts and flushes for region at 1732447015016Disabling writes for close at 1732447015017 (+1 ms)Obtaining lock to block concurrent updates at 1732447015017Preparing flush snapshotting stores in 58548ec0d65011347d1619ef7b0b7484 at 1732447015017Finished memstore snapshotting TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732447015017Flushing stores of TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484. at 1732447015018 (+1 ms)Flushing 58548ec0d65011347d1619ef7b0b7484/A: creating writer at 1732447015018Flushing 58548ec0d65011347d1619ef7b0b7484/A: appending metadata at 1732447015022 (+4 ms)Flushing 58548ec0d65011347d1619ef7b0b7484/A: closing flushed file at 1732447015022Flushing 58548ec0d65011347d1619ef7b0b7484/B: creating writer at 1732447015440 (+418 ms)Flushing 58548ec0d65011347d1619ef7b0b7484/B: appending metadata at 1732447015444 (+4 ms)Flushing 58548ec0d65011347d1619ef7b0b7484/B: closing flushed file at 1732447015444Flushing 58548ec0d65011347d1619ef7b0b7484/C: creating writer at 1732447015859 (+415 ms)Flushing 58548ec0d65011347d1619ef7b0b7484/C: appending metadata at 1732447015864 (+5 ms)Flushing 58548ec0d65011347d1619ef7b0b7484/C: closing flushed file at 1732447015864Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e0d9373: reopening flushed file at 1732447016278 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa3bc64: reopening flushed file at 1732447016283 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7def0ecf: reopening flushed file at 1732447016288 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 58548ec0d65011347d1619ef7b0b7484 in 1277ms, sequenceid=405, compaction requested=true at 1732447016294 (+6 ms)Writing region close event to WAL at 1732447016387 (+93 ms)Running coprocessor post-close hooks at 1732447016390 (+3 ms)Closed at 1732447016390 2024-11-24T11:16:56,392 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:56,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=58548ec0d65011347d1619ef7b0b7484, regionState=CLOSED 2024-11-24T11:16:56,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:16:56,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-24T11:16:56,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 58548ec0d65011347d1619ef7b0b7484, server=f5f73e76a839,45939,1732446856176 in 1.5300 sec 2024-11-24T11:16:56,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-24T11:16:56,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58548ec0d65011347d1619ef7b0b7484, UNASSIGN in 1.5340 sec 2024-11-24T11:16:56,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-24T11:16:56,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-24T11:16:56,399 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732447016399"}]},"ts":"1732447016399"} 2024-11-24T11:16:56,400 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T11:16:56,400 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T11:16:56,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5440 sec 2024-11-24T11:16:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T11:16:57,015 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:57,016 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-24T11:16:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:57,020 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T11:16:57,022 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:57,024 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:57,026 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/recovered.edits] 2024-11-24T11:16:57,028 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9fbf0551d89e4523a9024c4b5c9bb350 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/9fbf0551d89e4523a9024c4b5c9bb350 2024-11-24T11:16:57,029 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/afd3c7b6d7e04ca0a21dacfe070d5733 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/afd3c7b6d7e04ca0a21dacfe070d5733 2024-11-24T11:16:57,030 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e1439df9e7534b9d84cee0c98cf2364e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/A/e1439df9e7534b9d84cee0c98cf2364e 2024-11-24T11:16:57,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4b5b436e549e44fb86ed96488a3f5dbe to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4b5b436e549e44fb86ed96488a3f5dbe 2024-11-24T11:16:57,034 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4e563d15142545a7979a76e3ede178fa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/4e563d15142545a7979a76e3ede178fa 2024-11-24T11:16:57,035 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a40c5189aeb34cbd8ca1ab90633f97f8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/B/a40c5189aeb34cbd8ca1ab90633f97f8 2024-11-24T11:16:57,038 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/55f2e79c04034579b752e0ae56653331 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/55f2e79c04034579b752e0ae56653331 2024-11-24T11:16:57,039 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/6b4ae1ee85d04950bfda18fb1b9c04ac to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/6b4ae1ee85d04950bfda18fb1b9c04ac 2024-11-24T11:16:57,041 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e1b2aacf39b6400cb57c5b1aa9202276 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/C/e1b2aacf39b6400cb57c5b1aa9202276 2024-11-24T11:16:57,044 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/recovered.edits/408.seqid to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484/recovered.edits/408.seqid 2024-11-24T11:16:57,044 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/58548ec0d65011347d1619ef7b0b7484 2024-11-24T11:16:57,045 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T11:16:57,047 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:57,050 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T11:16:57,052 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T11:16:57,054 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:57,054 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T11:16:57,054 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732447017054"}]},"ts":"9223372036854775807"} 2024-11-24T11:16:57,057 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T11:16:57,057 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 58548ec0d65011347d1619ef7b0b7484, NAME => 'TestAcidGuarantees,,1732446981494.58548ec0d65011347d1619ef7b0b7484.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T11:16:57,058 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T11:16:57,058 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732447017058"}]},"ts":"9223372036854775807"} 2024-11-24T11:16:57,060 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T11:16:57,060 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:57,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-24T11:16:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T11:16:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T11:16:57,136 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:57,149 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 250), OpenFileDescriptor=441 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 378), ProcessCount=11 (was 11), AvailableMemoryMB=5923 (was 6124) 2024-11-24T11:16:57,159 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=355, ProcessCount=11, AvailableMemoryMB=5923 2024-11-24T11:16:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:16:57,160 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:16:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:57,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T11:16:57,162 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:57,162 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-24T11:16:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-24T11:16:57,163 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T11:16:57,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742360_1536 (size=1061) 2024-11-24T11:16:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-24T11:16:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-24T11:16:57,574 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c 2024-11-24T11:16:57,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742361_1537 (size=53) 2024-11-24T11:16:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-24T11:16:57,986 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:16:57,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fc46f0a8c7ecfcf606c3547de7a28906, disabling compactions & flushes 2024-11-24T11:16:57,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:57,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:57,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. after waiting 0 ms 2024-11-24T11:16:57,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:57,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:57,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fc46f0a8c7ecfcf606c3547de7a28906: Waiting for close lock at 1732447017987Disabling compacts and flushes for region at 1732447017987Disabling writes for close at 1732447017987Writing region close event to WAL at 1732447017987Closed at 1732447017987 2024-11-24T11:16:57,990 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T11:16:57,991 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732447017990"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732447017990"}]},"ts":"1732447017990"} 2024-11-24T11:16:57,993 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T11:16:57,994 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T11:16:57,994 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732447017994"}]},"ts":"1732447017994"} 2024-11-24T11:16:57,995 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T11:16:57,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, ASSIGN}] 2024-11-24T11:16:57,997 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, ASSIGN 2024-11-24T11:16:57,998 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, ASSIGN; state=OFFLINE, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=false 2024-11-24T11:16:58,149 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:58,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, ASSIGN because future has completed 2024-11-24T11:16:58,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:16:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-24T11:16:58,312 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:58,312 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:16:58,313 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,313 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:16:58,313 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,313 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,315 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,317 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:16:58,317 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc46f0a8c7ecfcf606c3547de7a28906 columnFamilyName A 2024-11-24T11:16:58,317 DEBUG [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:58,318 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(327): Store=fc46f0a8c7ecfcf606c3547de7a28906/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:16:58,318 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,319 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:16:58,320 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc46f0a8c7ecfcf606c3547de7a28906 columnFamilyName B 2024-11-24T11:16:58,320 DEBUG [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:58,321 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(327): Store=fc46f0a8c7ecfcf606c3547de7a28906/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:16:58,321 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,322 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:16:58,322 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc46f0a8c7ecfcf606c3547de7a28906 columnFamilyName C 2024-11-24T11:16:58,323 DEBUG [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:16:58,323 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(327): Store=fc46f0a8c7ecfcf606c3547de7a28906/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:16:58,323 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:58,324 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,325 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,325 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,327 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,327 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,327 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:16:58,329 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,332 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T11:16:58,332 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened fc46f0a8c7ecfcf606c3547de7a28906; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70617805, jitterRate=0.05228729546070099}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:16:58,332 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:58,333 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for fc46f0a8c7ecfcf606c3547de7a28906: Running coprocessor pre-open hook at 1732447018313Writing region info on filesystem at 1732447018313Initializing all the Stores at 1732447018314 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732447018314Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732447018314Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732447018315 (+1 ms)Cleaning up temporary data from old regions at 1732447018327 (+12 ms)Running coprocessor post-open hooks at 1732447018332 (+5 ms)Region opened successfully at 1732447018333 (+1 ms) 2024-11-24T11:16:58,333 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., pid=140, masterSystemTime=1732447018308 2024-11-24T11:16:58,335 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:58,335 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:58,335 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=OPEN, openSeqNum=2, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:58,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:16:58,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-24T11:16:58,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 in 183 msec 2024-11-24T11:16:58,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-24T11:16:58,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, ASSIGN in 342 msec 2024-11-24T11:16:58,340 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T11:16:58,340 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732447018340"}]},"ts":"1732447018340"} 2024-11-24T11:16:58,341 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T11:16:58,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T11:16:58,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-24T11:16:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-24T11:16:59,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:16:59,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eedb90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:59,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:16:59,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:16:59,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:16:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:16:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:16:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3180fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:16:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:16:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:16:59,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:16:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25300f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:16:59,325 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:16:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:16:59,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:16:59,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:16:59,327 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T11:16:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f5f73e76a839,40345,1732446855400 2024-11-24T11:16:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42e5d6 2024-11-24T11:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T11:16:59,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T11:16:59,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T11:16:59,329 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T11:16:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T11:16:59,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742362_1538 (size=1097) 2024-11-24T11:16:59,787 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T11:16:59,787 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T11:16:59,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:16:59,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, REOPEN/MOVE}] 2024-11-24T11:16:59,797 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, REOPEN/MOVE 2024-11-24T11:16:59,797 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:16:59,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, REOPEN/MOVE because future has completed 2024-11-24T11:16:59,799 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:16:59,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:16:59,952 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:59,952 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:16:59,953 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing fc46f0a8c7ecfcf606c3547de7a28906, disabling compactions & flushes 2024-11-24T11:16:59,953 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:59,953 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:59,953 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. after waiting 0 ms 2024-11-24T11:16:59,953 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:59,963 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T11:16:59,964 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:16:59,964 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for fc46f0a8c7ecfcf606c3547de7a28906: Waiting for close lock at 1732447019953Running coprocessor pre-close hooks at 1732447019953Disabling compacts and flushes for region at 1732447019953Disabling writes for close at 1732447019953Writing region close event to WAL at 1732447019954 (+1 ms)Running coprocessor post-close hooks at 1732447019963 (+9 ms)Closed at 1732447019963 2024-11-24T11:16:59,964 WARN [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: fc46f0a8c7ecfcf606c3547de7a28906 to self. 2024-11-24T11:16:59,965 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:16:59,965 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=CLOSED 2024-11-24T11:16:59,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:16:59,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-24T11:16:59,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 in 168 msec 2024-11-24T11:16:59,969 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, REOPEN/MOVE; state=CLOSED, location=f5f73e76a839,45939,1732446856176; forceNewPlan=false, retain=true 2024-11-24T11:17:00,120 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=OPENING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, REOPEN/MOVE because future has completed 2024-11-24T11:17:00,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:17:00,279 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,279 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} 2024-11-24T11:17:00,279 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,279 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T11:17:00,280 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,280 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,281 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,281 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:17:00,282 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc46f0a8c7ecfcf606c3547de7a28906 columnFamilyName A 2024-11-24T11:17:00,283 DEBUG [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:00,283 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(327): Store=fc46f0a8c7ecfcf606c3547de7a28906/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:17:00,284 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,284 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:17:00,285 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc46f0a8c7ecfcf606c3547de7a28906 columnFamilyName B 2024-11-24T11:17:00,285 DEBUG [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:00,285 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(327): Store=fc46f0a8c7ecfcf606c3547de7a28906/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:17:00,285 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,286 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T11:17:00,286 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc46f0a8c7ecfcf606c3547de7a28906 columnFamilyName C 2024-11-24T11:17:00,286 DEBUG [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:00,287 INFO [StoreOpener-fc46f0a8c7ecfcf606c3547de7a28906-1 {}] regionserver.HStore(327): Store=fc46f0a8c7ecfcf606c3547de7a28906/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T11:17:00,287 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,287 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,288 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,289 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,290 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,290 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,291 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T11:17:00,292 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,293 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened fc46f0a8c7ecfcf606c3547de7a28906; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74303543, jitterRate=0.10720907151699066}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T11:17:00,293 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,294 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for fc46f0a8c7ecfcf606c3547de7a28906: Running coprocessor pre-open hook at 1732447020280Writing region info on filesystem at 1732447020280Initializing all the Stores at 1732447020280Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732447020280Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732447020281 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732447020281Cleaning up temporary data from old regions at 1732447020290 (+9 ms)Running coprocessor post-open hooks at 1732447020293 (+3 ms)Region opened successfully at 1732447020294 (+1 ms) 2024-11-24T11:17:00,294 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., pid=145, masterSystemTime=1732447020276 2024-11-24T11:17:00,296 DEBUG [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,296 INFO [RS_OPEN_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,297 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=OPEN, openSeqNum=5, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:17:00,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-24T11:17:00,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 in 177 msec 2024-11-24T11:17:00,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-24T11:17:00,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, REOPEN/MOVE in 507 msec 2024-11-24T11:17:00,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-24T11:17:00,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-24T11:17:00,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 979 msec 2024-11-24T11:17:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T11:17:00,312 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:00,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bc30a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,313 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1781d435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62cbd9a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227ae0e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,317 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b23c61a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,318 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c273c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401e798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,320 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3954c1cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49484119, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2ee11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403c2d82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9da94f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cbdd75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,329 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7050f2d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,330 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237f255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1001aa8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,333 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5549a343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@660382ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16487325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,336 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e167ab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,338 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8ceda0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5129a55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,340 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6417b312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,342 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa1fc52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f0ee4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,344 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e023000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb84b60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf3480a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5f73e76a839,40345,-1 for getting cluster id 2024-11-24T11:17:00,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T11:17:00,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e11dde-e3cf-4444-a74a-bbf00db082b1' 2024-11-24T11:17:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T11:17:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e11dde-e3cf-4444-a74a-bbf00db082b1" 2024-11-24T11:17:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c0b715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5f73e76a839,40345,-1] 2024-11-24T11:17:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T11:17:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:00,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T11:17:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef370a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T11:17:00,348 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,348 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,348 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,349 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,349 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,349 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,350 DEBUG [Thread-2424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,350 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,351 DEBUG [Thread-2413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,351 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T11:17:00,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,352 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:00,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5f73e76a839,45939,1732446856176, seqNum=-1] 2024-11-24T11:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T11:17:00,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-24T11:17:00,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T11:17:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T11:17:00,354 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,355 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:00,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5] 2024-11-24T11:17:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:17:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:00,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:00,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b39883457e834554a2551b5026d60823_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447020358/Put/seqid=0 2024-11-24T11:17:00,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732447020379, max timestamp=1732447020386Num uniques 859; 2024-11-24T11:17:00,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:17:00,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732447020379, max timestamp=1732447020386Num uniques 859; 2024-11-24T11:17:00,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:17:00,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732447020379, max timestamp=1732447020386Num uniques 859; 2024-11-24T11:17:00,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:17:00,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742363_1539 (size=17034) 2024-11-24T11:17:00,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447080400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447080401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447080404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447080405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447080406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T11:17:00,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447080514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447080514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447080514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447080514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447080514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T11:17:00,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447080724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447080724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447080724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447080724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447080724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:00,794 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:00,797 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b39883457e834554a2551b5026d60823_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b39883457e834554a2551b5026d60823_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:00,798 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/55b3e7ae107c4dce850d79d8efa481fd, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:00,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/55b3e7ae107c4dce850d79d8efa481fd is 175, key is test_row_0/A:col10/1732447020358/Put/seqid=0 2024-11-24T11:17:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742364_1540 (size=48139) 2024-11-24T11:17:00,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:00,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T11:17:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447081033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447081034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447081034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447081034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447081034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,201 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/55b3e7ae107c4dce850d79d8efa481fd 2024-11-24T11:17:01,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/9a7cc0a190e940f9bb01a3161d3bfcdb is 50, key is test_row_0/B:col10/1732447020358/Put/seqid=0 2024-11-24T11:17:01,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742365_1541 (size=12001) 2024-11-24T11:17:01,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/9a7cc0a190e940f9bb01a3161d3bfcdb 2024-11-24T11:17:01,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/168867cb7ab5455487bb5f197aeb31fa is 50, key is test_row_0/C:col10/1732447020358/Put/seqid=0 2024-11-24T11:17:01,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742366_1542 (size=12001) 2024-11-24T11:17:01,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T11:17:01,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447081544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447081544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447081544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447081544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447081553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:01,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:01,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/168867cb7ab5455487bb5f197aeb31fa 2024-11-24T11:17:01,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/55b3e7ae107c4dce850d79d8efa481fd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/55b3e7ae107c4dce850d79d8efa481fd 2024-11-24T11:17:01,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/55b3e7ae107c4dce850d79d8efa481fd, entries=250, sequenceid=15, filesize=47.0 K 2024-11-24T11:17:01,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/9a7cc0a190e940f9bb01a3161d3bfcdb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/9a7cc0a190e940f9bb01a3161d3bfcdb 2024-11-24T11:17:01,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/9a7cc0a190e940f9bb01a3161d3bfcdb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T11:17:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/168867cb7ab5455487bb5f197aeb31fa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/168867cb7ab5455487bb5f197aeb31fa 2024-11-24T11:17:01,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/168867cb7ab5455487bb5f197aeb31fa, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T11:17:01,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fc46f0a8c7ecfcf606c3547de7a28906 in 1292ms, sequenceid=15, compaction requested=false 2024-11-24T11:17:01,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:01,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-24T11:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:01,729 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T11:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ccdf5841d0944683b1dfe5e51152a922_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447020402/Put/seqid=0 2024-11-24T11:17:01,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742367_1543 (size=12154) 2024-11-24T11:17:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:01,749 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ccdf5841d0944683b1dfe5e51152a922_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ccdf5841d0944683b1dfe5e51152a922_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:01,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/c00775628d8b4b8dbb9a4e466670c8f4, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:01,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/c00775628d8b4b8dbb9a4e466670c8f4 is 175, key is test_row_0/A:col10/1732447020402/Put/seqid=0 2024-11-24T11:17:01,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742368_1544 (size=30955) 2024-11-24T11:17:01,906 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T11:17:02,153 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/c00775628d8b4b8dbb9a4e466670c8f4 2024-11-24T11:17:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a828e5b4da394adca481354831b9ce8d is 50, key is test_row_0/B:col10/1732447020402/Put/seqid=0 2024-11-24T11:17:02,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742369_1545 (size=12001) 2024-11-24T11:17:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T11:17:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:02,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:02,573 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a828e5b4da394adca481354831b9ce8d 2024-11-24T11:17:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/07e02b19bbc5451a94cc4da4f5a28653 is 50, key is test_row_0/C:col10/1732447020402/Put/seqid=0 2024-11-24T11:17:02,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742370_1546 (size=12001) 2024-11-24T11:17:02,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447082564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447082563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,598 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447082598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447082598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447082598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447082703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447082713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447082714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447082714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447082913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447082924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447082924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447082924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:02,986 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/07e02b19bbc5451a94cc4da4f5a28653 2024-11-24T11:17:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/c00775628d8b4b8dbb9a4e466670c8f4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/c00775628d8b4b8dbb9a4e466670c8f4 2024-11-24T11:17:02,992 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/c00775628d8b4b8dbb9a4e466670c8f4, entries=150, sequenceid=50, filesize=30.2 K 2024-11-24T11:17:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a828e5b4da394adca481354831b9ce8d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a828e5b4da394adca481354831b9ce8d 2024-11-24T11:17:02,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a828e5b4da394adca481354831b9ce8d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T11:17:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/07e02b19bbc5451a94cc4da4f5a28653 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/07e02b19bbc5451a94cc4da4f5a28653 2024-11-24T11:17:02,998 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/07e02b19bbc5451a94cc4da4f5a28653, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T11:17:02,999 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=67.09 KB/68700 for fc46f0a8c7ecfcf606c3547de7a28906 in 1269ms, sequenceid=50, compaction requested=false 2024-11-24T11:17:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-24T11:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-24T11:17:03,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-24T11:17:03,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6450 sec 2024-11-24T11:17:03,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.6500 sec 2024-11-24T11:17:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:03,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:17:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:03,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112456e1136213a943b680cafa477b5929ae_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447022564/Put/seqid=0 2024-11-24T11:17:03,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742371_1547 (size=14594) 2024-11-24T11:17:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447083247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447083248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447083251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447083252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447083363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447083364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447083364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447083364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447083573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447083573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447083574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447083574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:03,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112456e1136213a943b680cafa477b5929ae_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456e1136213a943b680cafa477b5929ae_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:03,642 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7610805c5c54487aab7e22cbc809bdd3, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:03,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7610805c5c54487aab7e22cbc809bdd3 is 175, key is test_row_0/A:col10/1732447022564/Put/seqid=0 2024-11-24T11:17:03,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742372_1548 (size=39549) 2024-11-24T11:17:03,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7610805c5c54487aab7e22cbc809bdd3 2024-11-24T11:17:03,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/fda36dc21903479b8b94006dc1b7195f is 50, key is test_row_0/B:col10/1732447022564/Put/seqid=0 2024-11-24T11:17:03,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742373_1549 (size=12001) 2024-11-24T11:17:03,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447083884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447083884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447083884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447083884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/fda36dc21903479b8b94006dc1b7195f 2024-11-24T11:17:04,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/31faf07306f742b0ac380fb29211f8fc is 50, key is test_row_0/C:col10/1732447022564/Put/seqid=0 2024-11-24T11:17:04,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742374_1550 (size=12001) 2024-11-24T11:17:04,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/31faf07306f742b0ac380fb29211f8fc 2024-11-24T11:17:04,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7610805c5c54487aab7e22cbc809bdd3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7610805c5c54487aab7e22cbc809bdd3 2024-11-24T11:17:04,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7610805c5c54487aab7e22cbc809bdd3, entries=200, sequenceid=64, filesize=38.6 K 2024-11-24T11:17:04,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/fda36dc21903479b8b94006dc1b7195f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fda36dc21903479b8b94006dc1b7195f 2024-11-24T11:17:04,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fda36dc21903479b8b94006dc1b7195f, entries=150, sequenceid=64, filesize=11.7 K 2024-11-24T11:17:04,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/31faf07306f742b0ac380fb29211f8fc as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/31faf07306f742b0ac380fb29211f8fc 2024-11-24T11:17:04,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/31faf07306f742b0ac380fb29211f8fc, entries=150, sequenceid=64, filesize=11.7 K 2024-11-24T11:17:04,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fc46f0a8c7ecfcf606c3547de7a28906 in 887ms, sequenceid=64, compaction requested=true 2024-11-24T11:17:04,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:04,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:17:04,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:04,113 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:04,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:17:04,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:04,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:17:04,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T11:17:04,114 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:04,114 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:04,114 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:04,114 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/B is initiating minor compaction (all files) 2024-11-24T11:17:04,114 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/A is initiating minor compaction (all files) 2024-11-24T11:17:04,114 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/B in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,114 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/A in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,114 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/9a7cc0a190e940f9bb01a3161d3bfcdb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a828e5b4da394adca481354831b9ce8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fda36dc21903479b8b94006dc1b7195f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=35.2 K 2024-11-24T11:17:04,114 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/55b3e7ae107c4dce850d79d8efa481fd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/c00775628d8b4b8dbb9a4e466670c8f4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7610805c5c54487aab7e22cbc809bdd3] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=115.9 K 2024-11-24T11:17:04,115 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,115 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/55b3e7ae107c4dce850d79d8efa481fd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/c00775628d8b4b8dbb9a4e466670c8f4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7610805c5c54487aab7e22cbc809bdd3] 2024-11-24T11:17:04,115 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a7cc0a190e940f9bb01a3161d3bfcdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732447020358 2024-11-24T11:17:04,115 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55b3e7ae107c4dce850d79d8efa481fd, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732447020355 2024-11-24T11:17:04,116 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting c00775628d8b4b8dbb9a4e466670c8f4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732447020402 2024-11-24T11:17:04,116 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a828e5b4da394adca481354831b9ce8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732447020402 2024-11-24T11:17:04,116 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7610805c5c54487aab7e22cbc809bdd3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732447022564 2024-11-24T11:17:04,116 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting fda36dc21903479b8b94006dc1b7195f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732447022564 2024-11-24T11:17:04,120 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:04,122 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124e8cd4bb302a4414f9d9c8aa8ccc39543_fc46f0a8c7ecfcf606c3547de7a28906 store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:04,122 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:04,122 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/7ecf7a4e58c44f96ad1181e191194b68 is 50, key is test_row_0/B:col10/1732447022564/Put/seqid=0 2024-11-24T11:17:04,123 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124e8cd4bb302a4414f9d9c8aa8ccc39543_fc46f0a8c7ecfcf606c3547de7a28906, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:04,123 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e8cd4bb302a4414f9d9c8aa8ccc39543_fc46f0a8c7ecfcf606c3547de7a28906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:04,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742376_1552 (size=4469) 2024-11-24T11:17:04,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742375_1551 (size=12104) 2024-11-24T11:17:04,132 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#A#compaction#468 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:04,132 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/667cb1ba955e403e8234f3baa9d36de2 is 175, key is test_row_0/A:col10/1732447022564/Put/seqid=0 2024-11-24T11:17:04,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742377_1553 (size=31058) 2024-11-24T11:17:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:04,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:17:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:04,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243a1e17d34797441a9fbb0b84526e29f1_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447024396/Put/seqid=0 2024-11-24T11:17:04,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742378_1554 (size=12154) 2024-11-24T11:17:04,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:04,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243a1e17d34797441a9fbb0b84526e29f1_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243a1e17d34797441a9fbb0b84526e29f1_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:04,411 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/a3cc368a1ce24d708e41fb7ac3650ce9, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:04,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/a3cc368a1ce24d708e41fb7ac3650ce9 is 175, key is test_row_0/A:col10/1732447024396/Put/seqid=0 2024-11-24T11:17:04,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447084409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447084411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447084411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447084412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742379_1555 (size=30955) 2024-11-24T11:17:04,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/a3cc368a1ce24d708e41fb7ac3650ce9 2024-11-24T11:17:04,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b0dc2f0f2b7b43caa47f0d17c686f1dd is 50, key is test_row_0/B:col10/1732447024396/Put/seqid=0 2024-11-24T11:17:04,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742380_1556 (size=12001) 2024-11-24T11:17:04,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b0dc2f0f2b7b43caa47f0d17c686f1dd 2024-11-24T11:17:04,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/82680ef026944db6b190d900fe65c58a is 50, key is test_row_0/C:col10/1732447024396/Put/seqid=0 2024-11-24T11:17:04,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742381_1557 (size=12001) 2024-11-24T11:17:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T11:17:04,513 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:04,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-24T11:17:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-24T11:17:04,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:04,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:04,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447084523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447084523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447084523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447084524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,535 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/7ecf7a4e58c44f96ad1181e191194b68 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/7ecf7a4e58c44f96ad1181e191194b68 2024-11-24T11:17:04,538 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/B of fc46f0a8c7ecfcf606c3547de7a28906 into 7ecf7a4e58c44f96ad1181e191194b68(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:04,538 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:04,538 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/B, priority=13, startTime=1732447024113; duration=0sec 2024-11-24T11:17:04,538 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:04,538 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:B 2024-11-24T11:17:04,538 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:04,539 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/667cb1ba955e403e8234f3baa9d36de2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/667cb1ba955e403e8234f3baa9d36de2 2024-11-24T11:17:04,539 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:04,539 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/C is initiating minor compaction (all files) 2024-11-24T11:17:04,539 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/C in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,539 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/168867cb7ab5455487bb5f197aeb31fa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/07e02b19bbc5451a94cc4da4f5a28653, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/31faf07306f742b0ac380fb29211f8fc] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=35.2 K 2024-11-24T11:17:04,540 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 168867cb7ab5455487bb5f197aeb31fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732447020358 2024-11-24T11:17:04,540 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e02b19bbc5451a94cc4da4f5a28653, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732447020402 2024-11-24T11:17:04,540 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 31faf07306f742b0ac380fb29211f8fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732447022564 2024-11-24T11:17:04,542 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/A of fc46f0a8c7ecfcf606c3547de7a28906 into 667cb1ba955e403e8234f3baa9d36de2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:04,542 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:04,542 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/A, priority=13, startTime=1732447024113; duration=0sec 2024-11-24T11:17:04,542 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:04,542 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:A 2024-11-24T11:17:04,545 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#C#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:04,545 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/d6ecd803a77b4a4ebb2cd7da14b3a8fa is 50, key is test_row_0/C:col10/1732447022564/Put/seqid=0 2024-11-24T11:17:04,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742382_1558 (size=12104) 2024-11-24T11:17:04,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447084613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,616 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-24T11:17:04,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-24T11:17:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447084733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447084733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447084733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447084733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:04,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-24T11:17:04,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:04,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:04,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-24T11:17:04,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/82680ef026944db6b190d900fe65c58a 2024-11-24T11:17:04,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/a3cc368a1ce24d708e41fb7ac3650ce9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/a3cc368a1ce24d708e41fb7ac3650ce9 2024-11-24T11:17:04,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/a3cc368a1ce24d708e41fb7ac3650ce9, entries=150, sequenceid=83, filesize=30.2 K 2024-11-24T11:17:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b0dc2f0f2b7b43caa47f0d17c686f1dd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0dc2f0f2b7b43caa47f0d17c686f1dd 2024-11-24T11:17:04,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0dc2f0f2b7b43caa47f0d17c686f1dd, entries=150, sequenceid=83, filesize=11.7 K 2024-11-24T11:17:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/82680ef026944db6b190d900fe65c58a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/82680ef026944db6b190d900fe65c58a 2024-11-24T11:17:04,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/82680ef026944db6b190d900fe65c58a, entries=150, sequenceid=83, filesize=11.7 K 2024-11-24T11:17:04,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for fc46f0a8c7ecfcf606c3547de7a28906 in 498ms, sequenceid=83, compaction requested=false 2024-11-24T11:17:04,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:04,952 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/d6ecd803a77b4a4ebb2cd7da14b3a8fa as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/d6ecd803a77b4a4ebb2cd7da14b3a8fa 2024-11-24T11:17:04,955 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/C of fc46f0a8c7ecfcf606c3547de7a28906 into d6ecd803a77b4a4ebb2cd7da14b3a8fa(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:04,955 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:04,955 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/C, priority=13, startTime=1732447024113; duration=0sec 2024-11-24T11:17:04,955 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:04,955 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:C 2024-11-24T11:17:04,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-24T11:17:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:04,974 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T11:17:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:04,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124547c701f50a148b1bef442766c80b785_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447024409/Put/seqid=0 2024-11-24T11:17:04,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742383_1559 (size=12154) 2024-11-24T11:17:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:04,985 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124547c701f50a148b1bef442766c80b785_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124547c701f50a148b1bef442766c80b785_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/2d74a01255e44671881bd6c2f734db5b, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/2d74a01255e44671881bd6c2f734db5b is 175, key is test_row_0/A:col10/1732447024409/Put/seqid=0 2024-11-24T11:17:04,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742384_1560 (size=30955) 2024-11-24T11:17:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:05,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:05,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447085062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447085067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447085068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447085068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-24T11:17:05,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447085173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447085173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447085184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447085184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447085383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447085383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,392 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/2d74a01255e44671881bd6c2f734db5b 2024-11-24T11:17:05,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447085394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447085394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/60cbafbe1a064420ab44bd1da472b927 is 50, key is test_row_0/B:col10/1732447024409/Put/seqid=0 2024-11-24T11:17:05,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742385_1561 (size=12001) 2024-11-24T11:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-24T11:17:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447085693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447085693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447085703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447085704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:05,801 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/60cbafbe1a064420ab44bd1da472b927 2024-11-24T11:17:05,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b7a1daa1d74442f5800b3723835f112d is 50, key is test_row_0/C:col10/1732447024409/Put/seqid=0 2024-11-24T11:17:05,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742386_1562 (size=12001) 2024-11-24T11:17:06,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447086203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447086203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:06,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447086213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447086214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:06,216 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b7a1daa1d74442f5800b3723835f112d 2024-11-24T11:17:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/2d74a01255e44671881bd6c2f734db5b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/2d74a01255e44671881bd6c2f734db5b 2024-11-24T11:17:06,222 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/2d74a01255e44671881bd6c2f734db5b, entries=150, sequenceid=100, filesize=30.2 K 2024-11-24T11:17:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/60cbafbe1a064420ab44bd1da472b927 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/60cbafbe1a064420ab44bd1da472b927 2024-11-24T11:17:06,225 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/60cbafbe1a064420ab44bd1da472b927, entries=150, sequenceid=100, filesize=11.7 K 2024-11-24T11:17:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b7a1daa1d74442f5800b3723835f112d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b7a1daa1d74442f5800b3723835f112d 2024-11-24T11:17:06,229 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b7a1daa1d74442f5800b3723835f112d, entries=150, sequenceid=100, filesize=11.7 K 2024-11-24T11:17:06,232 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for fc46f0a8c7ecfcf606c3547de7a28906 in 1258ms, sequenceid=100, compaction requested=true 2024-11-24T11:17:06,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:06,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:06,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-24T11:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-24T11:17:06,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-24T11:17:06,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-24T11:17:06,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.7210 sec 2024-11-24T11:17:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-24T11:17:06,664 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:06,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-24T11:17:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T11:17:06,666 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:06,666 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:06,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T11:17:06,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T11:17:06,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:06,818 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T11:17:06,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:06,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:06,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:06,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:06,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:06,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:06,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c730bb26e2be45f18fe02508c84e7301_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447025066/Put/seqid=0 2024-11-24T11:17:06,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742387_1563 (size=12154) 2024-11-24T11:17:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T11:17:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:07,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:07,231 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c730bb26e2be45f18fe02508c84e7301_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c730bb26e2be45f18fe02508c84e7301_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:07,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1f3f768774d34fc9b0a499ed39758360, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:07,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1f3f768774d34fc9b0a499ed39758360 is 175, key is test_row_0/A:col10/1732447025066/Put/seqid=0 2024-11-24T11:17:07,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742388_1564 (size=30955) 2024-11-24T11:17:07,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447087233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447087237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,240 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447087240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447087241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T11:17:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447087343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447087354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447087354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447087553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447087563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447087564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,635 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1f3f768774d34fc9b0a499ed39758360 2024-11-24T11:17:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/fd4533e65bc04066947b45c98514cce6 is 50, key is test_row_0/B:col10/1732447025066/Put/seqid=0 2024-11-24T11:17:07,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742389_1565 (size=12001) 2024-11-24T11:17:07,674 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/fd4533e65bc04066947b45c98514cce6 2024-11-24T11:17:07,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/57b2bd151c9b4ea4bc582f802436a9dc is 50, key is test_row_0/C:col10/1732447025066/Put/seqid=0 2024-11-24T11:17:07,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742390_1566 (size=12001) 2024-11-24T11:17:07,690 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/57b2bd151c9b4ea4bc582f802436a9dc 2024-11-24T11:17:07,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1f3f768774d34fc9b0a499ed39758360 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1f3f768774d34fc9b0a499ed39758360 2024-11-24T11:17:07,710 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1f3f768774d34fc9b0a499ed39758360, entries=150, sequenceid=119, filesize=30.2 K 2024-11-24T11:17:07,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/fd4533e65bc04066947b45c98514cce6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fd4533e65bc04066947b45c98514cce6 2024-11-24T11:17:07,717 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fd4533e65bc04066947b45c98514cce6, entries=150, sequenceid=119, filesize=11.7 K 2024-11-24T11:17:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/57b2bd151c9b4ea4bc582f802436a9dc as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/57b2bd151c9b4ea4bc582f802436a9dc 2024-11-24T11:17:07,724 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/57b2bd151c9b4ea4bc582f802436a9dc, entries=150, sequenceid=119, filesize=11.7 K 2024-11-24T11:17:07,725 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for fc46f0a8c7ecfcf606c3547de7a28906 in 907ms, sequenceid=119, compaction requested=true 2024-11-24T11:17:07,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:07,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:07,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-24T11:17:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-24T11:17:07,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-24T11:17:07,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0600 sec 2024-11-24T11:17:07,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.0650 sec 2024-11-24T11:17:07,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T11:17:07,804 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:07,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-24T11:17:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T11:17:07,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:07,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:07,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:07,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:17:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:07,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246f0256bcd2b0436696aa16a274a0bcb8_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447027236/Put/seqid=0 2024-11-24T11:17:07,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742391_1567 (size=14794) 2024-11-24T11:17:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447087893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447087894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447087895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T11:17:07,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:07,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:07,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447088003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447088004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447088004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T11:17:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447088214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447088214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447088214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:08,280 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246f0256bcd2b0436696aa16a274a0bcb8_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246f0256bcd2b0436696aa16a274a0bcb8_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:08,281 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0991caa00191466ab59508b3cdf9efb7, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:08,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0991caa00191466ab59508b3cdf9efb7 is 175, key is test_row_0/A:col10/1732447027236/Put/seqid=0 2024-11-24T11:17:08,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742392_1568 (size=39749) 2024-11-24T11:17:08,291 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0991caa00191466ab59508b3cdf9efb7 2024-11-24T11:17:08,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/f5133d7c9fb0467fbc768a0f5de6d880 is 50, key is test_row_0/B:col10/1732447027236/Put/seqid=0 2024-11-24T11:17:08,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742393_1569 (size=12151) 2024-11-24T11:17:08,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:08,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:08,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T11:17:08,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447088523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447088523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447088523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447088643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:08,647 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:08,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/f5133d7c9fb0467fbc768a0f5de6d880 2024-11-24T11:17:08,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/6ab52cc006da4546bc802ba1b09baf76 is 50, key is test_row_0/C:col10/1732447027236/Put/seqid=0 2024-11-24T11:17:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742394_1570 (size=12151) 2024-11-24T11:17:08,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:08,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:08,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:08,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:08,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:08,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T11:17:09,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:09,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:09,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:09,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:09,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:09,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:09,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447089033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:09,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447089033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447089033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:09,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/6ab52cc006da4546bc802ba1b09baf76 2024-11-24T11:17:09,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0991caa00191466ab59508b3cdf9efb7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0991caa00191466ab59508b3cdf9efb7 2024-11-24T11:17:09,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0991caa00191466ab59508b3cdf9efb7, entries=200, sequenceid=134, filesize=38.8 K 2024-11-24T11:17:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/f5133d7c9fb0467fbc768a0f5de6d880 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/f5133d7c9fb0467fbc768a0f5de6d880 2024-11-24T11:17:09,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/f5133d7c9fb0467fbc768a0f5de6d880, entries=150, sequenceid=134, filesize=11.9 K 2024-11-24T11:17:09,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/6ab52cc006da4546bc802ba1b09baf76 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6ab52cc006da4546bc802ba1b09baf76 2024-11-24T11:17:09,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6ab52cc006da4546bc802ba1b09baf76, entries=150, sequenceid=134, filesize=11.9 K 2024-11-24T11:17:09,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fc46f0a8c7ecfcf606c3547de7a28906 in 1258ms, sequenceid=134, compaction requested=true 2024-11-24T11:17:09,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:09,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:17:09,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:09,124 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T11:17:09,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:17:09,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:09,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:17:09,125 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T11:17:09,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60258 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163672 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/A is initiating minor compaction (all files) 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/B is initiating minor compaction (all files) 2024-11-24T11:17:09,126 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/B in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:09,126 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/A in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:09,126 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/7ecf7a4e58c44f96ad1181e191194b68, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0dc2f0f2b7b43caa47f0d17c686f1dd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/60cbafbe1a064420ab44bd1da472b927, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fd4533e65bc04066947b45c98514cce6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/f5133d7c9fb0467fbc768a0f5de6d880] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=58.8 K 2024-11-24T11:17:09,126 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/667cb1ba955e403e8234f3baa9d36de2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/a3cc368a1ce24d708e41fb7ac3650ce9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/2d74a01255e44671881bd6c2f734db5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1f3f768774d34fc9b0a499ed39758360, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0991caa00191466ab59508b3cdf9efb7] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=159.8 K 2024-11-24T11:17:09,126 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/667cb1ba955e403e8234f3baa9d36de2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/a3cc368a1ce24d708e41fb7ac3650ce9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/2d74a01255e44671881bd6c2f734db5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1f3f768774d34fc9b0a499ed39758360, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0991caa00191466ab59508b3cdf9efb7] 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ecf7a4e58c44f96ad1181e191194b68, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732447022564 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 667cb1ba955e403e8234f3baa9d36de2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732447022564 2024-11-24T11:17:09,126 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b0dc2f0f2b7b43caa47f0d17c686f1dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732447023249 2024-11-24T11:17:09,127 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3cc368a1ce24d708e41fb7ac3650ce9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732447023249 2024-11-24T11:17:09,127 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 60cbafbe1a064420ab44bd1da472b927, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732447024402 2024-11-24T11:17:09,127 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d74a01255e44671881bd6c2f734db5b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732447024402 2024-11-24T11:17:09,127 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting fd4533e65bc04066947b45c98514cce6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732447025066 2024-11-24T11:17:09,127 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f3f768774d34fc9b0a499ed39758360, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732447025066 2024-11-24T11:17:09,127 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0991caa00191466ab59508b3cdf9efb7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732447027236 2024-11-24T11:17:09,127 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting f5133d7c9fb0467fbc768a0f5de6d880, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732447027236 2024-11-24T11:17:09,133 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:09,134 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124a978279bc3c44f30a5cc5e091a201119_fc46f0a8c7ecfcf606c3547de7a28906 store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:09,135 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:09,136 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/1955c3ed1e9a406daa9082660ac35ac5 is 50, key is test_row_0/B:col10/1732447027236/Put/seqid=0 2024-11-24T11:17:09,137 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124a978279bc3c44f30a5cc5e091a201119_fc46f0a8c7ecfcf606c3547de7a28906, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:09,137 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a978279bc3c44f30a5cc5e091a201119_fc46f0a8c7ecfcf606c3547de7a28906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:09,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742395_1571 (size=12425) 2024-11-24T11:17:09,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742396_1572 (size=4469) 2024-11-24T11:17:09,148 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#A#compaction#484 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:09,148 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/212c10e9648a4cd6a0ebb6a572ea7020 is 175, key is test_row_0/A:col10/1732447027236/Put/seqid=0 2024-11-24T11:17:09,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742397_1573 (size=31379) 2024-11-24T11:17:09,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T11:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:09,181 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:09,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112452317ccf668343eaa2687f718264776f_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447027893/Put/seqid=0 2024-11-24T11:17:09,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742398_1574 (size=12304) 2024-11-24T11:17:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:09,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:09,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447089327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:09,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447089434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:09,550 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/1955c3ed1e9a406daa9082660ac35ac5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/1955c3ed1e9a406daa9082660ac35ac5 2024-11-24T11:17:09,553 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/B of fc46f0a8c7ecfcf606c3547de7a28906 into 1955c3ed1e9a406daa9082660ac35ac5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:09,553 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:09,553 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/B, priority=11, startTime=1732447029124; duration=0sec 2024-11-24T11:17:09,553 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:09,553 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:B 2024-11-24T11:17:09,553 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T11:17:09,555 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60258 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T11:17:09,555 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/C is initiating minor compaction (all files) 2024-11-24T11:17:09,555 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/C in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:09,556 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/d6ecd803a77b4a4ebb2cd7da14b3a8fa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/82680ef026944db6b190d900fe65c58a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b7a1daa1d74442f5800b3723835f112d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/57b2bd151c9b4ea4bc582f802436a9dc, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6ab52cc006da4546bc802ba1b09baf76] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=58.8 K 2024-11-24T11:17:09,556 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ecd803a77b4a4ebb2cd7da14b3a8fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732447022564 2024-11-24T11:17:09,556 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/212c10e9648a4cd6a0ebb6a572ea7020 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/212c10e9648a4cd6a0ebb6a572ea7020 2024-11-24T11:17:09,556 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 82680ef026944db6b190d900fe65c58a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732447023249 2024-11-24T11:17:09,556 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a1daa1d74442f5800b3723835f112d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732447024402 2024-11-24T11:17:09,556 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 57b2bd151c9b4ea4bc582f802436a9dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732447025066 2024-11-24T11:17:09,557 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab52cc006da4546bc802ba1b09baf76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732447027236 2024-11-24T11:17:09,559 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/A of fc46f0a8c7ecfcf606c3547de7a28906 into 212c10e9648a4cd6a0ebb6a572ea7020(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:09,559 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:09,559 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/A, priority=11, startTime=1732447029124; duration=0sec 2024-11-24T11:17:09,559 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:09,559 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:A 2024-11-24T11:17:09,565 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#C#compaction#486 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:09,565 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/af3d001c8d21457ea6dcb34a8582fd32 is 50, key is test_row_0/C:col10/1732447027236/Put/seqid=0 2024-11-24T11:17:09,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742399_1575 (size=12425) 2024-11-24T11:17:09,571 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/af3d001c8d21457ea6dcb34a8582fd32 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/af3d001c8d21457ea6dcb34a8582fd32 2024-11-24T11:17:09,574 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/C of fc46f0a8c7ecfcf606c3547de7a28906 into af3d001c8d21457ea6dcb34a8582fd32(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:09,574 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:09,574 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/C, priority=11, startTime=1732447029125; duration=0sec 2024-11-24T11:17:09,574 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:09,574 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:C 2024-11-24T11:17:09,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:09,597 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112452317ccf668343eaa2687f718264776f_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112452317ccf668343eaa2687f718264776f_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:09,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aac5ee84601a4cedb26b2310d6c2efbe, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aac5ee84601a4cedb26b2310d6c2efbe is 175, key is test_row_0/A:col10/1732447027893/Put/seqid=0 2024-11-24T11:17:09,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742400_1576 (size=31105) 2024-11-24T11:17:09,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447089643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T11:17:09,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447089953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:10,001 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aac5ee84601a4cedb26b2310d6c2efbe 2024-11-24T11:17:10,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/8c403502b670400f900cc2ef00422e90 is 50, key is test_row_0/B:col10/1732447027893/Put/seqid=0 2024-11-24T11:17:10,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742401_1577 (size=12151) 2024-11-24T11:17:10,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447090043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:10,046 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447090053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447090053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:10,055 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:10,056 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:10,409 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/8c403502b670400f900cc2ef00422e90 2024-11-24T11:17:10,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/003fb6aeccb549fb84dd70033995ffd2 is 50, key is test_row_0/C:col10/1732447027893/Put/seqid=0 2024-11-24T11:17:10,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742402_1578 (size=12151) 2024-11-24T11:17:10,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447090463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:10,818 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/003fb6aeccb549fb84dd70033995ffd2 2024-11-24T11:17:10,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aac5ee84601a4cedb26b2310d6c2efbe as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aac5ee84601a4cedb26b2310d6c2efbe 2024-11-24T11:17:10,824 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aac5ee84601a4cedb26b2310d6c2efbe, entries=150, sequenceid=151, filesize=30.4 K 2024-11-24T11:17:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/8c403502b670400f900cc2ef00422e90 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/8c403502b670400f900cc2ef00422e90 2024-11-24T11:17:10,827 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/8c403502b670400f900cc2ef00422e90, entries=150, sequenceid=151, filesize=11.9 K 2024-11-24T11:17:10,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/003fb6aeccb549fb84dd70033995ffd2 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/003fb6aeccb549fb84dd70033995ffd2 2024-11-24T11:17:10,830 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/003fb6aeccb549fb84dd70033995ffd2, entries=150, sequenceid=151, filesize=11.9 K 2024-11-24T11:17:10,831 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fc46f0a8c7ecfcf606c3547de7a28906 in 1650ms, sequenceid=151, compaction requested=false 2024-11-24T11:17:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-24T11:17:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-24T11:17:10,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-24T11:17:10,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0250 sec 2024-11-24T11:17:10,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0300 sec 2024-11-24T11:17:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:11,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:17:11,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:11,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:11,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:11,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:11,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:11,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:11,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e1254ed974494495b752b83eb22d27cb_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447031485/Put/seqid=0 2024-11-24T11:17:11,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742403_1579 (size=14794) 2024-11-24T11:17:11,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:11,499 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e1254ed974494495b752b83eb22d27cb_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e1254ed974494495b752b83eb22d27cb_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:11,500 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/704f11a3636f45bca42c678067c57254, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:11,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/704f11a3636f45bca42c678067c57254 is 175, key is test_row_0/A:col10/1732447031485/Put/seqid=0 2024-11-24T11:17:11,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742404_1580 (size=39749) 2024-11-24T11:17:11,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/704f11a3636f45bca42c678067c57254 2024-11-24T11:17:11,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/6e0f3eae44e24ab195e1291f855a546c is 50, key is test_row_0/B:col10/1732447031485/Put/seqid=0 2024-11-24T11:17:11,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742405_1581 (size=12151) 2024-11-24T11:17:11,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447091541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:11,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:11,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:11,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447091653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:11,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447091864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:11,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/6e0f3eae44e24ab195e1291f855a546c 2024-11-24T11:17:11,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/1d1d936d3cc340db9de120daee44cf70 is 50, key is test_row_0/C:col10/1732447031485/Put/seqid=0 2024-11-24T11:17:11,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742406_1582 (size=12151) 2024-11-24T11:17:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T11:17:11,964 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:11,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-24T11:17:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T11:17:11,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:11,966 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:11,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:12,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447092063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:12,066 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T11:17:12,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447092073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:12,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447092074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:12,074 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_0' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:12,075 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:12,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T11:17:12,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:12,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:12,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447092174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:12,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T11:17:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T11:17:12,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/1d1d936d3cc340db9de120daee44cf70 2024-11-24T11:17:12,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/704f11a3636f45bca42c678067c57254 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/704f11a3636f45bca42c678067c57254 2024-11-24T11:17:12,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/704f11a3636f45bca42c678067c57254, entries=200, sequenceid=170, filesize=38.8 K 2024-11-24T11:17:12,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/6e0f3eae44e24ab195e1291f855a546c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/6e0f3eae44e24ab195e1291f855a546c 2024-11-24T11:17:12,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/6e0f3eae44e24ab195e1291f855a546c, entries=150, sequenceid=170, filesize=11.9 K 2024-11-24T11:17:12,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/1d1d936d3cc340db9de120daee44cf70 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/1d1d936d3cc340db9de120daee44cf70 2024-11-24T11:17:12,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/1d1d936d3cc340db9de120daee44cf70, entries=150, sequenceid=170, filesize=11.9 K 2024-11-24T11:17:12,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc46f0a8c7ecfcf606c3547de7a28906 in 853ms, sequenceid=170, compaction requested=true 2024-11-24T11:17:12,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:12,340 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:17:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:12,341 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:17:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:17:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:12,343 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:12,343 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/A is initiating minor compaction (all files) 2024-11-24T11:17:12,343 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/A in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,343 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/212c10e9648a4cd6a0ebb6a572ea7020, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aac5ee84601a4cedb26b2310d6c2efbe, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/704f11a3636f45bca42c678067c57254] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=99.8 K 2024-11-24T11:17:12,344 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,344 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/212c10e9648a4cd6a0ebb6a572ea7020, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aac5ee84601a4cedb26b2310d6c2efbe, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/704f11a3636f45bca42c678067c57254] 2024-11-24T11:17:12,344 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:12,344 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/B is initiating minor compaction (all files) 2024-11-24T11:17:12,344 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/B in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,344 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/1955c3ed1e9a406daa9082660ac35ac5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/8c403502b670400f900cc2ef00422e90, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/6e0f3eae44e24ab195e1291f855a546c] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=35.9 K 2024-11-24T11:17:12,345 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 212c10e9648a4cd6a0ebb6a572ea7020, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732447027236 2024-11-24T11:17:12,345 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1955c3ed1e9a406daa9082660ac35ac5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732447027236 2024-11-24T11:17:12,345 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac5ee84601a4cedb26b2310d6c2efbe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732447027884 2024-11-24T11:17:12,345 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c403502b670400f900cc2ef00422e90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732447027884 2024-11-24T11:17:12,345 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 704f11a3636f45bca42c678067c57254, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732447029308 2024-11-24T11:17:12,346 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0f3eae44e24ab195e1291f855a546c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732447029320 2024-11-24T11:17:12,361 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:12,373 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:12,373 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/719e69c7ecd846b795de9f1df0a6ef40 is 50, key is test_row_0/B:col10/1732447031485/Put/seqid=0 2024-11-24T11:17:12,374 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124f7f200ddcc9d4f8ebc1e054a85a5d68b_fc46f0a8c7ecfcf606c3547de7a28906 store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:12,376 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124f7f200ddcc9d4f8ebc1e054a85a5d68b_fc46f0a8c7ecfcf606c3547de7a28906, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:12,376 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f7f200ddcc9d4f8ebc1e054a85a5d68b_fc46f0a8c7ecfcf606c3547de7a28906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:12,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742408_1584 (size=4469) 2024-11-24T11:17:12,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742407_1583 (size=12527) 2024-11-24T11:17:12,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T11:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,424 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f91d1c39db494046876721c27cc57d95_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447031535/Put/seqid=0 2024-11-24T11:17:12,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742409_1585 (size=12304) 2024-11-24T11:17:12,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:12,444 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f91d1c39db494046876721c27cc57d95_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f91d1c39db494046876721c27cc57d95_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/f01bf9b24554475ab539961345c687cf, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/f01bf9b24554475ab539961345c687cf is 175, key is test_row_0/A:col10/1732447031535/Put/seqid=0 2024-11-24T11:17:12,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742410_1586 (size=31105) 2024-11-24T11:17:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T11:17:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:12,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:12,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447092759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:12,783 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:12,784 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/b6f4f977c5ef46e7ba22785df34921e0 is 175, key is test_row_0/A:col10/1732447031485/Put/seqid=0 2024-11-24T11:17:12,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742411_1587 (size=31481) 2024-11-24T11:17:12,795 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/719e69c7ecd846b795de9f1df0a6ef40 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/719e69c7ecd846b795de9f1df0a6ef40 2024-11-24T11:17:12,795 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/b6f4f977c5ef46e7ba22785df34921e0 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b6f4f977c5ef46e7ba22785df34921e0 2024-11-24T11:17:12,801 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/B of fc46f0a8c7ecfcf606c3547de7a28906 into 719e69c7ecd846b795de9f1df0a6ef40(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:12,801 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:12,802 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/B, priority=13, startTime=1732447032341; duration=0sec 2024-11-24T11:17:12,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:12,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:B 2024-11-24T11:17:12,802 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/A of fc46f0a8c7ecfcf606c3547de7a28906 into b6f4f977c5ef46e7ba22785df34921e0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:12,802 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:12,802 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:12,802 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/A, priority=13, startTime=1732447032340; duration=0sec 2024-11-24T11:17:12,802 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:12,802 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:A 2024-11-24T11:17:12,803 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:12,803 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/C is initiating minor compaction (all files) 2024-11-24T11:17:12,803 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/C in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:12,804 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/af3d001c8d21457ea6dcb34a8582fd32, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/003fb6aeccb549fb84dd70033995ffd2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/1d1d936d3cc340db9de120daee44cf70] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=35.9 K 2024-11-24T11:17:12,804 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting af3d001c8d21457ea6dcb34a8582fd32, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732447027236 2024-11-24T11:17:12,804 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 003fb6aeccb549fb84dd70033995ffd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732447027884 2024-11-24T11:17:12,805 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d1d936d3cc340db9de120daee44cf70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732447029320 2024-11-24T11:17:12,813 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:12,814 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/c9fe447872f4420792e63ee69921cc52 is 50, key is test_row_0/C:col10/1732447031485/Put/seqid=0 2024-11-24T11:17:12,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742412_1588 (size=12527) 2024-11-24T11:17:12,848 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/f01bf9b24554475ab539961345c687cf 2024-11-24T11:17:12,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b0ad62fc2f7747db84f8d04f0a65f109 is 50, key is test_row_0/B:col10/1732447031535/Put/seqid=0 2024-11-24T11:17:12,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447092873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:12,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:12,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:12,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742413_1589 (size=12151) 2024-11-24T11:17:13,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447093086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T11:17:13,232 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/c9fe447872f4420792e63ee69921cc52 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c9fe447872f4420792e63ee69921cc52 2024-11-24T11:17:13,237 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/C of fc46f0a8c7ecfcf606c3547de7a28906 into c9fe447872f4420792e63ee69921cc52(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:13,237 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:13,237 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/C, priority=13, startTime=1732447032341; duration=0sec 2024-11-24T11:17:13,238 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:13,238 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:C 2024-11-24T11:17:13,282 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b0ad62fc2f7747db84f8d04f0a65f109 2024-11-24T11:17:13,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/205da77fccec410e8397cb729d2d8d35 is 50, key is test_row_0/C:col10/1732447031535/Put/seqid=0 2024-11-24T11:17:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742414_1590 (size=12151) 2024-11-24T11:17:13,299 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/205da77fccec410e8397cb729d2d8d35 2024-11-24T11:17:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/f01bf9b24554475ab539961345c687cf as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/f01bf9b24554475ab539961345c687cf 2024-11-24T11:17:13,311 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/f01bf9b24554475ab539961345c687cf, entries=150, sequenceid=186, filesize=30.4 K 2024-11-24T11:17:13,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b0ad62fc2f7747db84f8d04f0a65f109 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0ad62fc2f7747db84f8d04f0a65f109 2024-11-24T11:17:13,315 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0ad62fc2f7747db84f8d04f0a65f109, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T11:17:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/205da77fccec410e8397cb729d2d8d35 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/205da77fccec410e8397cb729d2d8d35 2024-11-24T11:17:13,320 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/205da77fccec410e8397cb729d2d8d35, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T11:17:13,321 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc46f0a8c7ecfcf606c3547de7a28906 in 897ms, sequenceid=186, compaction requested=false 2024-11-24T11:17:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-24T11:17:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-24T11:17:13,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-24T11:17:13,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3630 sec 2024-11-24T11:17:13,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.3680 sec 2024-11-24T11:17:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:13,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:17:13,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:13,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:13,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:13,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:13,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:13,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:13,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dd1b90e393ca4bad85c2f288e648a3f8_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447032757/Put/seqid=0 2024-11-24T11:17:13,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742415_1591 (size=14794) 2024-11-24T11:17:13,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447093463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:13,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447093574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:13,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447093783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:13,846 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:13,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dd1b90e393ca4bad85c2f288e648a3f8_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dd1b90e393ca4bad85c2f288e648a3f8_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:13,856 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0ad5599cfea6489e8672dddab55f14a7, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:13,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0ad5599cfea6489e8672dddab55f14a7 is 175, key is test_row_0/A:col10/1732447032757/Put/seqid=0 2024-11-24T11:17:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742416_1592 (size=39749) 2024-11-24T11:17:13,877 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0ad5599cfea6489e8672dddab55f14a7 2024-11-24T11:17:13,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a76e53335b184233b2ce3255199fafdb is 50, key is test_row_0/B:col10/1732447032757/Put/seqid=0 2024-11-24T11:17:13,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742417_1593 (size=12151) 2024-11-24T11:17:13,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a76e53335b184233b2ce3255199fafdb 2024-11-24T11:17:13,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/526840c295e84dd7bf4d3295e712852d is 50, key is test_row_0/C:col10/1732447032757/Put/seqid=0 2024-11-24T11:17:13,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742418_1594 (size=12151) 2024-11-24T11:17:13,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/526840c295e84dd7bf4d3295e712852d 2024-11-24T11:17:13,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/0ad5599cfea6489e8672dddab55f14a7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0ad5599cfea6489e8672dddab55f14a7 2024-11-24T11:17:13,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0ad5599cfea6489e8672dddab55f14a7, entries=200, sequenceid=206, filesize=38.8 K 2024-11-24T11:17:13,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a76e53335b184233b2ce3255199fafdb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a76e53335b184233b2ce3255199fafdb 2024-11-24T11:17:13,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a76e53335b184233b2ce3255199fafdb, entries=150, sequenceid=206, filesize=11.9 K 2024-11-24T11:17:13,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/526840c295e84dd7bf4d3295e712852d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/526840c295e84dd7bf4d3295e712852d 2024-11-24T11:17:13,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/526840c295e84dd7bf4d3295e712852d, entries=150, sequenceid=206, filesize=11.9 K 2024-11-24T11:17:13,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fc46f0a8c7ecfcf606c3547de7a28906 in 569ms, sequenceid=206, compaction requested=true 2024-11-24T11:17:13,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:17:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:13,968 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:13,968 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:17:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:17:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:13,968 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:13,968 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:13,968 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/A is initiating minor compaction (all files) 2024-11-24T11:17:13,968 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/B is initiating minor compaction (all files) 2024-11-24T11:17:13,969 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/A in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:13,969 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/B in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:13,969 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/719e69c7ecd846b795de9f1df0a6ef40, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0ad62fc2f7747db84f8d04f0a65f109, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a76e53335b184233b2ce3255199fafdb] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=36.0 K 2024-11-24T11:17:13,969 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b6f4f977c5ef46e7ba22785df34921e0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/f01bf9b24554475ab539961345c687cf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0ad5599cfea6489e8672dddab55f14a7] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=99.9 K 2024-11-24T11:17:13,969 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:13,969 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b6f4f977c5ef46e7ba22785df34921e0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/f01bf9b24554475ab539961345c687cf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0ad5599cfea6489e8672dddab55f14a7] 2024-11-24T11:17:13,969 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f4f977c5ef46e7ba22785df34921e0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732447029320 2024-11-24T11:17:13,969 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 719e69c7ecd846b795de9f1df0a6ef40, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732447029320 2024-11-24T11:17:13,969 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting f01bf9b24554475ab539961345c687cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732447031516 2024-11-24T11:17:13,969 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ad62fc2f7747db84f8d04f0a65f109, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732447031516 2024-11-24T11:17:13,969 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad5599cfea6489e8672dddab55f14a7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732447032724 2024-11-24T11:17:13,969 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a76e53335b184233b2ce3255199fafdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732447032724 2024-11-24T11:17:13,975 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:13,976 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246ebc0ebbec624be8ba0130184016fc49_fc46f0a8c7ecfcf606c3547de7a28906 store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:13,977 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:13,978 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/73b2f990a87640cf92dbc27e4cbdc2a9 is 50, key is test_row_0/B:col10/1732447032757/Put/seqid=0 2024-11-24T11:17:13,978 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246ebc0ebbec624be8ba0130184016fc49_fc46f0a8c7ecfcf606c3547de7a28906, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:13,978 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246ebc0ebbec624be8ba0130184016fc49_fc46f0a8c7ecfcf606c3547de7a28906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:14,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742420_1596 (size=4469) 2024-11-24T11:17:14,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742419_1595 (size=12629) 2024-11-24T11:17:14,038 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/73b2f990a87640cf92dbc27e4cbdc2a9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/73b2f990a87640cf92dbc27e4cbdc2a9 2024-11-24T11:17:14,042 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/B of fc46f0a8c7ecfcf606c3547de7a28906 into 73b2f990a87640cf92dbc27e4cbdc2a9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:14,042 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:14,042 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/B, priority=13, startTime=1732447033968; duration=0sec 2024-11-24T11:17:14,043 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:14,043 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:B 2024-11-24T11:17:14,043 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:14,044 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:14,044 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/C is initiating minor compaction (all files) 2024-11-24T11:17:14,045 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/C in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,045 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c9fe447872f4420792e63ee69921cc52, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/205da77fccec410e8397cb729d2d8d35, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/526840c295e84dd7bf4d3295e712852d] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=36.0 K 2024-11-24T11:17:14,045 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c9fe447872f4420792e63ee69921cc52, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732447029320 2024-11-24T11:17:14,045 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 205da77fccec410e8397cb729d2d8d35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732447031516 2024-11-24T11:17:14,045 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 526840c295e84dd7bf4d3295e712852d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732447032724 2024-11-24T11:17:14,053 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:14,053 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/c7003f30b8fe458581969263f92b2e2f is 50, key is test_row_0/C:col10/1732447032757/Put/seqid=0 2024-11-24T11:17:14,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742421_1597 (size=12629) 2024-11-24T11:17:14,084 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/c7003f30b8fe458581969263f92b2e2f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c7003f30b8fe458581969263f92b2e2f 2024-11-24T11:17:14,091 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/C of fc46f0a8c7ecfcf606c3547de7a28906 into c7003f30b8fe458581969263f92b2e2f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:14,091 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:14,091 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/C, priority=13, startTime=1732447033968; duration=0sec 2024-11-24T11:17:14,091 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:14,091 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:C 2024-11-24T11:17:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:14,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:17:14,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:14,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:14,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:14,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:14,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:14,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:14,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d531d3e71204474bbbe85cbf0564459e_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447033456/Put/seqid=0 2024-11-24T11:17:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T11:17:14,114 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:14,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-24T11:17:14,116 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:14,117 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:14,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T11:17:14,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742422_1598 (size=12304) 2024-11-24T11:17:14,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:14,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d531d3e71204474bbbe85cbf0564459e_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d531d3e71204474bbbe85cbf0564459e_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:14,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/32283add7ba64ff78249fe5f276e0fbf, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:14,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/32283add7ba64ff78249fe5f276e0fbf is 175, key is test_row_0/A:col10/1732447033456/Put/seqid=0 2024-11-24T11:17:14,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742423_1599 (size=31105) 2024-11-24T11:17:14,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/32283add7ba64ff78249fe5f276e0fbf 2024-11-24T11:17:14,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/491f70eae9074434bcc802e4ba04d306 is 50, key is test_row_0/B:col10/1732447033456/Put/seqid=0 2024-11-24T11:17:14,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447094172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:14,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742424_1600 (size=12151) 2024-11-24T11:17:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T11:17:14,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T11:17:14,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447094284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:14,414 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:14,414 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/5fd9e22b72744552825f2ce4bb1c5421 is 175, key is test_row_0/A:col10/1732447032757/Put/seqid=0 2024-11-24T11:17:14,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742425_1601 (size=31583) 2024-11-24T11:17:14,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T11:17:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T11:17:14,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447094493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:14,520 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T11:17:14,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T11:17:14,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/491f70eae9074434bcc802e4ba04d306 2024-11-24T11:17:14,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/ce486dbd319c48d3a18be81ba00ba136 is 50, key is test_row_0/C:col10/1732447033456/Put/seqid=0 2024-11-24T11:17:14,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742426_1602 (size=12151) 2024-11-24T11:17:14,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T11:17:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T11:17:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447094804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:14,826 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/5fd9e22b72744552825f2ce4bb1c5421 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/5fd9e22b72744552825f2ce4bb1c5421 2024-11-24T11:17:14,837 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/A of fc46f0a8c7ecfcf606c3547de7a28906 into 5fd9e22b72744552825f2ce4bb1c5421(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:14,838 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:14,838 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/A, priority=13, startTime=1732447033967; duration=0sec 2024-11-24T11:17:14,838 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:14,838 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:A 2024-11-24T11:17:14,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T11:17:14,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:14,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:14,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:15,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/ce486dbd319c48d3a18be81ba00ba136 2024-11-24T11:17:15,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/32283add7ba64ff78249fe5f276e0fbf as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32283add7ba64ff78249fe5f276e0fbf 2024-11-24T11:17:15,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32283add7ba64ff78249fe5f276e0fbf, entries=150, sequenceid=224, filesize=30.4 K 2024-11-24T11:17:15,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/491f70eae9074434bcc802e4ba04d306 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/491f70eae9074434bcc802e4ba04d306 2024-11-24T11:17:15,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/491f70eae9074434bcc802e4ba04d306, entries=150, sequenceid=224, filesize=11.9 K 2024-11-24T11:17:15,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/ce486dbd319c48d3a18be81ba00ba136 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/ce486dbd319c48d3a18be81ba00ba136 2024-11-24T11:17:15,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/ce486dbd319c48d3a18be81ba00ba136, entries=150, sequenceid=224, filesize=11.9 K 2024-11-24T11:17:15,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc46f0a8c7ecfcf606c3547de7a28906 in 924ms, sequenceid=224, compaction requested=false 2024-11-24T11:17:15,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:15,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T11:17:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:15,036 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T11:17:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c1473e565bc94ef08bab259ff602cedd_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447034170/Put/seqid=0 2024-11-24T11:17:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742427_1603 (size=12304) 2024-11-24T11:17:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:15,076 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c1473e565bc94ef08bab259ff602cedd_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c1473e565bc94ef08bab259ff602cedd_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:15,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/d90d1411bdac4e389e9900698b451915, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:15,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/d90d1411bdac4e389e9900698b451915 is 175, key is test_row_0/A:col10/1732447034170/Put/seqid=0 2024-11-24T11:17:15,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742428_1604 (size=31105) 2024-11-24T11:17:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T11:17:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:15,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447095365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447095473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:15,486 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/d90d1411bdac4e389e9900698b451915 2024-11-24T11:17:15,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b6e516ee2888424fa8df1683f6b35e8a is 50, key is test_row_0/B:col10/1732447034170/Put/seqid=0 2024-11-24T11:17:15,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742429_1605 (size=12151) 2024-11-24T11:17:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447095684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:15,894 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b6e516ee2888424fa8df1683f6b35e8a 2024-11-24T11:17:15,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b6c15997ba3d4a7aad404eabe844046d is 50, key is test_row_0/C:col10/1732447034170/Put/seqid=0 2024-11-24T11:17:15,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742430_1606 (size=12151) 2024-11-24T11:17:15,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b6c15997ba3d4a7aad404eabe844046d 2024-11-24T11:17:15,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/d90d1411bdac4e389e9900698b451915 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/d90d1411bdac4e389e9900698b451915 2024-11-24T11:17:15,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/d90d1411bdac4e389e9900698b451915, entries=150, sequenceid=241, filesize=30.4 K 2024-11-24T11:17:15,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/b6e516ee2888424fa8df1683f6b35e8a as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b6e516ee2888424fa8df1683f6b35e8a 2024-11-24T11:17:15,926 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b6e516ee2888424fa8df1683f6b35e8a, entries=150, sequenceid=241, filesize=11.9 K 2024-11-24T11:17:15,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b6c15997ba3d4a7aad404eabe844046d as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b6c15997ba3d4a7aad404eabe844046d 2024-11-24T11:17:15,931 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b6c15997ba3d4a7aad404eabe844046d, entries=150, sequenceid=241, filesize=11.9 K 2024-11-24T11:17:15,931 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fc46f0a8c7ecfcf606c3547de7a28906 in 895ms, sequenceid=241, compaction requested=true 2024-11-24T11:17:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:15,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:15,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-24T11:17:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-24T11:17:15,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-24T11:17:15,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-11-24T11:17:15,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8210 sec 2024-11-24T11:17:15,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T11:17:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:16,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b2f88bad6a4a42b58f08d84ef1e28f14_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447035337/Put/seqid=0 2024-11-24T11:17:16,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742431_1607 (size=14844) 2024-11-24T11:17:16,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447096090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447096093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59478 deadline: 1732447096114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59516 deadline: 1732447096114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,117 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_2' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,118 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447096204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447096204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T11:17:16,264 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:16,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-24T11:17:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T11:17:16,267 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:16,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:16,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T11:17:16,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447096414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447096414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T11:17:16,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:16,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:16,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b2f88bad6a4a42b58f08d84ef1e28f14_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b2f88bad6a4a42b58f08d84ef1e28f14_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:16,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7903488656104e18a7bf97d29c68bdc7, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:16,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7903488656104e18a7bf97d29c68bdc7 is 175, key is test_row_0/A:col10/1732447035337/Put/seqid=0 2024-11-24T11:17:16,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742432_1608 (size=39799) 2024-11-24T11:17:16,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T11:17:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T11:17:16,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T11:17:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447096724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447096724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:16,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7903488656104e18a7bf97d29c68bdc7 2024-11-24T11:17:16,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/67ea8e240f114c8d8685bb95d23a9252 is 50, key is test_row_0/B:col10/1732447035337/Put/seqid=0 2024-11-24T11:17:16,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742433_1609 (size=12201) 2024-11-24T11:17:16,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/67ea8e240f114c8d8685bb95d23a9252 2024-11-24T11:17:16,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/54698cfac4b24d3381779afcba0e581c is 50, key is test_row_0/C:col10/1732447035337/Put/seqid=0 2024-11-24T11:17:16,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742434_1610 (size=12201) 2024-11-24T11:17:16,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T11:17:16,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:16,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:16,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T11:17:17,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T11:17:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:17,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T11:17:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:17,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447097234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:17,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447097234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:17,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/54698cfac4b24d3381779afcba0e581c 2024-11-24T11:17:17,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/7903488656104e18a7bf97d29c68bdc7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7903488656104e18a7bf97d29c68bdc7 2024-11-24T11:17:17,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7903488656104e18a7bf97d29c68bdc7, entries=200, sequenceid=258, filesize=38.9 K 2024-11-24T11:17:17,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/67ea8e240f114c8d8685bb95d23a9252 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/67ea8e240f114c8d8685bb95d23a9252 2024-11-24T11:17:17,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/67ea8e240f114c8d8685bb95d23a9252, entries=150, sequenceid=258, filesize=11.9 K 2024-11-24T11:17:17,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/54698cfac4b24d3381779afcba0e581c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/54698cfac4b24d3381779afcba0e581c 2024-11-24T11:17:17,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/54698cfac4b24d3381779afcba0e581c, entries=150, sequenceid=258, filesize=11.9 K 2024-11-24T11:17:17,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fc46f0a8c7ecfcf606c3547de7a28906 in 1290ms, sequenceid=258, compaction requested=true 2024-11-24T11:17:17,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:17,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:17:17,288 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:17:17,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:17:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:17,289 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:17:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:17:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:17,289 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:17:17,289 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:17:17,290 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/A is initiating minor compaction (all files) 2024-11-24T11:17:17,290 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/B is initiating minor compaction (all files) 2024-11-24T11:17:17,290 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/B in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,290 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/A in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,290 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/5fd9e22b72744552825f2ce4bb1c5421, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32283add7ba64ff78249fe5f276e0fbf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/d90d1411bdac4e389e9900698b451915, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7903488656104e18a7bf97d29c68bdc7] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=130.5 K 2024-11-24T11:17:17,290 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/73b2f990a87640cf92dbc27e4cbdc2a9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/491f70eae9074434bcc802e4ba04d306, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b6e516ee2888424fa8df1683f6b35e8a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/67ea8e240f114c8d8685bb95d23a9252] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=48.0 K 2024-11-24T11:17:17,290 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,290 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/5fd9e22b72744552825f2ce4bb1c5421, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32283add7ba64ff78249fe5f276e0fbf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/d90d1411bdac4e389e9900698b451915, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7903488656104e18a7bf97d29c68bdc7] 2024-11-24T11:17:17,291 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b2f990a87640cf92dbc27e4cbdc2a9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732447032724 2024-11-24T11:17:17,291 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd9e22b72744552825f2ce4bb1c5421, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732447032724 2024-11-24T11:17:17,291 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 491f70eae9074434bcc802e4ba04d306, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732447033456 2024-11-24T11:17:17,291 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32283add7ba64ff78249fe5f276e0fbf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732447033456 2024-11-24T11:17:17,291 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90d1411bdac4e389e9900698b451915, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732447034157 2024-11-24T11:17:17,291 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e516ee2888424fa8df1683f6b35e8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732447034157 2024-11-24T11:17:17,292 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7903488656104e18a7bf97d29c68bdc7, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732447035337 2024-11-24T11:17:17,292 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ea8e240f114c8d8685bb95d23a9252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732447035337 2024-11-24T11:17:17,300 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:17,301 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:17,301 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/afeeb3200c0949b99219e0be99824ca1 is 50, key is test_row_0/B:col10/1732447035337/Put/seqid=0 2024-11-24T11:17:17,304 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124937979dae561446383884927cdbe9d6a_fc46f0a8c7ecfcf606c3547de7a28906 store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:17,307 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124937979dae561446383884927cdbe9d6a_fc46f0a8c7ecfcf606c3547de7a28906, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:17,307 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124937979dae561446383884927cdbe9d6a_fc46f0a8c7ecfcf606c3547de7a28906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:17,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742436_1612 (size=4469) 2024-11-24T11:17:17,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T11:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,338 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T11:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:17,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742435_1611 (size=12815) 2024-11-24T11:17:17,364 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/afeeb3200c0949b99219e0be99824ca1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/afeeb3200c0949b99219e0be99824ca1 2024-11-24T11:17:17,368 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/B of fc46f0a8c7ecfcf606c3547de7a28906 into afeeb3200c0949b99219e0be99824ca1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:17,368 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:17,368 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/B, priority=12, startTime=1732447037289; duration=0sec 2024-11-24T11:17:17,369 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:17,369 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:B 2024-11-24T11:17:17,369 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T11:17:17,370 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T11:17:17,370 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/C is initiating minor compaction (all files) 2024-11-24T11:17:17,371 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/C in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,371 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c7003f30b8fe458581969263f92b2e2f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/ce486dbd319c48d3a18be81ba00ba136, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b6c15997ba3d4a7aad404eabe844046d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/54698cfac4b24d3381779afcba0e581c] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=48.0 K 2024-11-24T11:17:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124240a43a624504f0980856bd4405958db_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447036088/Put/seqid=0 2024-11-24T11:17:17,373 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting c7003f30b8fe458581969263f92b2e2f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732447032724 2024-11-24T11:17:17,373 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting ce486dbd319c48d3a18be81ba00ba136, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732447033456 2024-11-24T11:17:17,374 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c15997ba3d4a7aad404eabe844046d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732447034157 2024-11-24T11:17:17,374 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 54698cfac4b24d3381779afcba0e581c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732447035337 2024-11-24T11:17:17,390 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:17,391 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/26ef15ce85244463bf3724f1c6399fc1 is 50, key is test_row_0/C:col10/1732447035337/Put/seqid=0 2024-11-24T11:17:17,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742437_1613 (size=12454) 2024-11-24T11:17:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,398 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124240a43a624504f0980856bd4405958db_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124240a43a624504f0980856bd4405958db_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/23e67b5c1ee449d48ffaa2b30d236ee4, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/23e67b5c1ee449d48ffaa2b30d236ee4 is 175, key is test_row_0/A:col10/1732447036088/Put/seqid=0 2024-11-24T11:17:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T11:17:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742438_1614 (size=12815) 2024-11-24T11:17:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742439_1615 (size=31255) 2024-11-24T11:17:17,409 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/23e67b5c1ee449d48ffaa2b30d236ee4 2024-11-24T11:17:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/e657f6521ed340e6b4d3622c175f9ebb is 50, key is test_row_0/B:col10/1732447036088/Put/seqid=0 2024-11-24T11:17:17,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742440_1616 (size=12301) 2024-11-24T11:17:17,426 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/e657f6521ed340e6b4d3622c175f9ebb 2024-11-24T11:17:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/e17ee16f543342c099ea068ced8b7cf9 is 50, key is test_row_0/C:col10/1732447036088/Put/seqid=0 2024-11-24T11:17:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742441_1617 (size=12301) 2024-11-24T11:17:17,462 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/e17ee16f543342c099ea068ced8b7cf9 2024-11-24T11:17:17,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/23e67b5c1ee449d48ffaa2b30d236ee4 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/23e67b5c1ee449d48ffaa2b30d236ee4 2024-11-24T11:17:17,475 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/23e67b5c1ee449d48ffaa2b30d236ee4, entries=150, sequenceid=273, filesize=30.5 K 2024-11-24T11:17:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/e657f6521ed340e6b4d3622c175f9ebb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/e657f6521ed340e6b4d3622c175f9ebb 2024-11-24T11:17:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,484 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/e657f6521ed340e6b4d3622c175f9ebb, entries=150, sequenceid=273, filesize=12.0 K 2024-11-24T11:17:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/e17ee16f543342c099ea068ced8b7cf9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e17ee16f543342c099ea068ced8b7cf9 2024-11-24T11:17:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e17ee16f543342c099ea068ced8b7cf9, entries=150, sequenceid=273, filesize=12.0 K 2024-11-24T11:17:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,490 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for fc46f0a8c7ecfcf606c3547de7a28906 in 152ms, sequenceid=273, compaction requested=false 2024-11-24T11:17:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-24T11:17:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-24T11:17:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-24T11:17:17,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2240 sec 2024-11-24T11:17:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.2310 sec 2024-11-24T11:17:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,737 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,738 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/8b5ca51989034cb4b2aaf1042081c01e is 175, key is test_row_0/A:col10/1732447035337/Put/seqid=0 2024-11-24T11:17:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742442_1618 (size=31769) 2024-11-24T11:17:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,812 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/26ef15ce85244463bf3724f1c6399fc1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/26ef15ce85244463bf3724f1c6399fc1 2024-11-24T11:17:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,817 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/C of fc46f0a8c7ecfcf606c3547de7a28906 into 26ef15ce85244463bf3724f1c6399fc1(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:17,817 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:17,817 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/C, priority=12, startTime=1732447037289; duration=0sec 2024-11-24T11:17:17,817 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:17,817 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:C 2024-11-24T11:17:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,189 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/8b5ca51989034cb4b2aaf1042081c01e as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/8b5ca51989034cb4b2aaf1042081c01e 2024-11-24T11:17:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/A of fc46f0a8c7ecfcf606c3547de7a28906 into 8b5ca51989034cb4b2aaf1042081c01e(size=31.0 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:18,193 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/A, priority=12, startTime=1732447037288; duration=0sec 2024-11-24T11:17:18,193 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:18,193 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:A 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:18,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T11:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124056db37c62254c3ea40f5e9f84a6cd67_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447038265/Put/seqid=0 2024-11-24T11:17:18,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742443_1619 (size=20074) 2024-11-24T11:17:18,319 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:18,322 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124056db37c62254c3ea40f5e9f84a6cd67_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124056db37c62254c3ea40f5e9f84a6cd67_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:18,323 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/b01fcb2011864d72969c7cd311629a28, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:18,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/b01fcb2011864d72969c7cd311629a28 is 175, key is test_row_0/A:col10/1732447038265/Put/seqid=0 2024-11-24T11:17:18,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742444_1620 (size=57333) 2024-11-24T11:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732447038337, max timestamp=1732447038347Num uniques 859; 2024-11-24T11:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732447038337, max timestamp=1732447038347Num uniques 859; 2024-11-24T11:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732447038337, max timestamp=1732447038347Num uniques 859; 2024-11-24T11:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T11:17:18,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447098414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447098416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T11:17:18,424 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:18,425 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-24T11:17:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-24T11:17:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T11:17:18,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T11:17:18,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T11:17:18,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T11:17:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447098524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447098524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T11:17:18,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T11:17:18,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:18,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:18,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:18,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T11:17:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T11:17:18,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/b01fcb2011864d72969c7cd311629a28 2024-11-24T11:17:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447098733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447098734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:18,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/61256eda4ce446ab86c33a50e516c1e9 is 50, key is test_row_0/B:col10/1732447038265/Put/seqid=0 2024-11-24T11:17:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T11:17:18,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59562 deadline: 1732447098753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:18,758 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f5f73e76a839,45939,1732446856176 for 'test_row_1' in fc46f0a8c7ecfcf606c3547de7a28906 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18352 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T11:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:18,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742445_1621 (size=12301) 2024-11-24T11:17:18,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/61256eda4ce446ab86c33a50e516c1e9 2024-11-24T11:17:18,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b3f9908b8cbc4632a518095d27506174 is 50, key is test_row_0/C:col10/1732447038265/Put/seqid=0 2024-11-24T11:17:18,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742446_1622 (size=12301) 2024-11-24T11:17:18,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b3f9908b8cbc4632a518095d27506174 2024-11-24T11:17:18,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/b01fcb2011864d72969c7cd311629a28 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b01fcb2011864d72969c7cd311629a28 2024-11-24T11:17:18,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b01fcb2011864d72969c7cd311629a28, entries=300, sequenceid=286, filesize=56.0 K 2024-11-24T11:17:18,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/61256eda4ce446ab86c33a50e516c1e9 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/61256eda4ce446ab86c33a50e516c1e9 2024-11-24T11:17:18,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/61256eda4ce446ab86c33a50e516c1e9, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T11:17:18,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/b3f9908b8cbc4632a518095d27506174 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b3f9908b8cbc4632a518095d27506174 2024-11-24T11:17:18,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b3f9908b8cbc4632a518095d27506174, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T11:17:18,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fc46f0a8c7ecfcf606c3547de7a28906 in 521ms, sequenceid=286, compaction requested=true 2024-11-24T11:17:18,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:18,820 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:18,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:17:18,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:18,821 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:18,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:17:18,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:18,821 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120357 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:18,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:17:18,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:18,821 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/A is initiating minor compaction (all files) 2024-11-24T11:17:18,821 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/A in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,821 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/8b5ca51989034cb4b2aaf1042081c01e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/23e67b5c1ee449d48ffaa2b30d236ee4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b01fcb2011864d72969c7cd311629a28] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=117.5 K 2024-11-24T11:17:18,821 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,822 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/8b5ca51989034cb4b2aaf1042081c01e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/23e67b5c1ee449d48ffaa2b30d236ee4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b01fcb2011864d72969c7cd311629a28] 2024-11-24T11:17:18,822 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b5ca51989034cb4b2aaf1042081c01e, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732447035337 2024-11-24T11:17:18,822 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:18,822 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/B is initiating minor compaction (all files) 2024-11-24T11:17:18,822 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/B in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,822 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/afeeb3200c0949b99219e0be99824ca1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/e657f6521ed340e6b4d3622c175f9ebb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/61256eda4ce446ab86c33a50e516c1e9] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=36.5 K 2024-11-24T11:17:18,823 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e67b5c1ee449d48ffaa2b30d236ee4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732447036074 2024-11-24T11:17:18,823 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting afeeb3200c0949b99219e0be99824ca1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732447035337 2024-11-24T11:17:18,823 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b01fcb2011864d72969c7cd311629a28, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732447038265 2024-11-24T11:17:18,823 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e657f6521ed340e6b4d3622c175f9ebb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732447036074 2024-11-24T11:17:18,824 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 61256eda4ce446ab86c33a50e516c1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732447038265 2024-11-24T11:17:18,829 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:18,842 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#B#compaction#523 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:18,843 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a968be65a33b41d5aa6c107f717c11dd is 50, key is test_row_0/B:col10/1732447038265/Put/seqid=0 2024-11-24T11:17:18,849 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112406377f6aed3b461c81f7b06a6c887e57_fc46f0a8c7ecfcf606c3547de7a28906 store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:18,851 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112406377f6aed3b461c81f7b06a6c887e57_fc46f0a8c7ecfcf606c3547de7a28906, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:18,851 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112406377f6aed3b461c81f7b06a6c887e57_fc46f0a8c7ecfcf606c3547de7a28906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:18,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742447_1623 (size=13017) 2024-11-24T11:17:18,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742448_1624 (size=4469) 2024-11-24T11:17:18,883 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#A#compaction#522 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:18,884 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/32059cb79721425791ceffa6621cef81 is 175, key is test_row_0/A:col10/1732447038265/Put/seqid=0 2024-11-24T11:17:18,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45939 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T11:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,885 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T11:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:18,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742449_1625 (size=31971) 2024-11-24T11:17:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112461a5a453d0194738b5f51d27875a0e42_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447038415/Put/seqid=0 2024-11-24T11:17:18,903 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/32059cb79721425791ceffa6621cef81 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32059cb79721425791ceffa6621cef81 2024-11-24T11:17:18,907 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/A of fc46f0a8c7ecfcf606c3547de7a28906 into 32059cb79721425791ceffa6621cef81(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:18,907 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:18,907 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/A, priority=13, startTime=1732447038820; duration=0sec 2024-11-24T11:17:18,907 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:18,907 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:A 2024-11-24T11:17:18,907 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:18,907 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:18,908 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/C is initiating minor compaction (all files) 2024-11-24T11:17:18,908 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/C in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:18,908 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/26ef15ce85244463bf3724f1c6399fc1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e17ee16f543342c099ea068ced8b7cf9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b3f9908b8cbc4632a518095d27506174] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=36.5 K 2024-11-24T11:17:18,908 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ef15ce85244463bf3724f1c6399fc1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732447035337 2024-11-24T11:17:18,908 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting e17ee16f543342c099ea068ced8b7cf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732447036074 2024-11-24T11:17:18,908 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3f9908b8cbc4632a518095d27506174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732447038265 2024-11-24T11:17:18,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742450_1626 (size=12454) 2024-11-24T11:17:18,915 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:18,915 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/e62a701eb95742edab4754996c8b92c7 is 50, key is test_row_0/C:col10/1732447038265/Put/seqid=0 2024-11-24T11:17:18,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742451_1627 (size=13017) 2024-11-24T11:17:18,932 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/e62a701eb95742edab4754996c8b92c7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e62a701eb95742edab4754996c8b92c7 2024-11-24T11:17:18,936 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/C of fc46f0a8c7ecfcf606c3547de7a28906 into e62a701eb95742edab4754996c8b92c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:18,936 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:18,936 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/C, priority=13, startTime=1732447038821; duration=0sec 2024-11-24T11:17:18,936 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:18,936 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:C 2024-11-24T11:17:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:19,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. as already flushing 2024-11-24T11:17:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T11:17:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447099058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447099062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447099173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447099173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,282 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/a968be65a33b41d5aa6c107f717c11dd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a968be65a33b41d5aa6c107f717c11dd 2024-11-24T11:17:19,286 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/B of fc46f0a8c7ecfcf606c3547de7a28906 into a968be65a33b41d5aa6c107f717c11dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:19,286 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:19,286 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/B, priority=13, startTime=1732447038821; duration=0sec 2024-11-24T11:17:19,286 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:19,286 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:B 2024-11-24T11:17:19,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:19,314 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112461a5a453d0194738b5f51d27875a0e42_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112461a5a453d0194738b5f51d27875a0e42_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:19,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aec62209bd7a44ba87d7362a755fd813, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:19,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aec62209bd7a44ba87d7362a755fd813 is 175, key is test_row_0/A:col10/1732447038415/Put/seqid=0 2024-11-24T11:17:19,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742452_1628 (size=31255) 2024-11-24T11:17:19,319 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aec62209bd7a44ba87d7362a755fd813 2024-11-24T11:17:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/308a89b48bd048e8b5a4ffddf4247897 is 50, key is test_row_0/B:col10/1732447038415/Put/seqid=0 2024-11-24T11:17:19,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742453_1629 (size=12301) 2024-11-24T11:17:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447099383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447099383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T11:17:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447099694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447099694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:19,735 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/308a89b48bd048e8b5a4ffddf4247897 2024-11-24T11:17:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/71c399ecc300400fb2c7254b595bc202 is 50, key is test_row_0/C:col10/1732447038415/Put/seqid=0 2024-11-24T11:17:19,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742454_1630 (size=12301) 2024-11-24T11:17:20,159 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/71c399ecc300400fb2c7254b595bc202 2024-11-24T11:17:20,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/aec62209bd7a44ba87d7362a755fd813 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aec62209bd7a44ba87d7362a755fd813 2024-11-24T11:17:20,165 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aec62209bd7a44ba87d7362a755fd813, entries=150, sequenceid=323, filesize=30.5 K 2024-11-24T11:17:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/308a89b48bd048e8b5a4ffddf4247897 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/308a89b48bd048e8b5a4ffddf4247897 2024-11-24T11:17:20,168 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/308a89b48bd048e8b5a4ffddf4247897, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T11:17:20,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/71c399ecc300400fb2c7254b595bc202 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/71c399ecc300400fb2c7254b595bc202 2024-11-24T11:17:20,176 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/71c399ecc300400fb2c7254b595bc202, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T11:17:20,177 INFO [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fc46f0a8c7ecfcf606c3547de7a28906 in 1292ms, sequenceid=323, compaction requested=false 2024-11-24T11:17:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5f73e76a839:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-24T11:17:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-24T11:17:20,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-24T11:17:20,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7520 sec 2024-11-24T11:17:20,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7680 sec 2024-11-24T11:17:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:20,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T11:17:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:20,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f0f551a05cc347b285d13eebdd395c06_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447039060/Put/seqid=0 2024-11-24T11:17:20,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742455_1631 (size=12454) 2024-11-24T11:17:20,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732447040249, max timestamp=1732447040263Num uniques 859; 2024-11-24T11:17:20,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732447040249, max timestamp=1732447040263Num uniques 859; 2024-11-24T11:17:20,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:17:20,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:17:20,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732447040249, max timestamp=1732447040263Num uniques 859; 2024-11-24T11:17:20,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f5f73e76a839:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T11:17:20,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447100327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447100329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:20,352 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-24T11:17:20,352 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:20,352 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,352 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:20,355 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-24T11:17:20,355 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:20,355 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,356 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:20,357 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-24T11:17:20,357 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:20,357 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,357 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:20,359 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-24T11:17:20,359 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:20,359 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,359 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:20,361 INFO [Thread-2424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2424. 2024-11-24T11:17:20,361 DEBUG [Thread-2424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:20,361 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,361 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:20,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:20,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447100443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447100443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T11:17:20,574 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:20,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:20,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f0f551a05cc347b285d13eebdd395c06_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f0f551a05cc347b285d13eebdd395c06_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:20,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/962b35c811dc4c95bc18f340ddd58599, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:20,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/962b35c811dc4c95bc18f340ddd58599 is 175, key is test_row_0/A:col10/1732447039060/Put/seqid=0 2024-11-24T11:17:20,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742456_1632 (size=31255) 2024-11-24T11:17:20,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447100653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447100654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447100965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447100965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:21,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/962b35c811dc4c95bc18f340ddd58599 2024-11-24T11:17:21,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/3ea2c9b28791448ba9fec492699e145f is 50, key is test_row_0/B:col10/1732447039060/Put/seqid=0 2024-11-24T11:17:21,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742457_1633 (size=12301) 2024-11-24T11:17:21,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/3ea2c9b28791448ba9fec492699e145f 2024-11-24T11:17:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T11:17:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59546 deadline: 1732447101474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59530 deadline: 1732447101474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 , the old value is region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T11:17:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., hostname=f5f73e76a839,45939,1732446856176, seqNum=5 because the exception is null or not the one we care about 2024-11-24T11:17:21,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/90cf3dcd385a4cd7be2f3856c59585e8 is 50, key is test_row_0/C:col10/1732447039060/Put/seqid=0 2024-11-24T11:17:21,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742458_1634 (size=12301) 2024-11-24T11:17:21,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/90cf3dcd385a4cd7be2f3856c59585e8 2024-11-24T11:17:21,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/962b35c811dc4c95bc18f340ddd58599 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/962b35c811dc4c95bc18f340ddd58599 2024-11-24T11:17:21,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/962b35c811dc4c95bc18f340ddd58599, entries=150, sequenceid=337, filesize=30.5 K 2024-11-24T11:17:21,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/3ea2c9b28791448ba9fec492699e145f as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/3ea2c9b28791448ba9fec492699e145f 2024-11-24T11:17:21,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/3ea2c9b28791448ba9fec492699e145f, entries=150, sequenceid=337, filesize=12.0 K 2024-11-24T11:17:21,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/90cf3dcd385a4cd7be2f3856c59585e8 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/90cf3dcd385a4cd7be2f3856c59585e8 2024-11-24T11:17:21,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/90cf3dcd385a4cd7be2f3856c59585e8, entries=150, sequenceid=337, filesize=12.0 K 2024-11-24T11:17:21,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fc46f0a8c7ecfcf606c3547de7a28906 in 1705ms, sequenceid=337, compaction requested=true 2024-11-24T11:17:21,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:21,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T11:17:21,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:21,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T11:17:21,910 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc46f0a8c7ecfcf606c3547de7a28906:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T11:17:21,911 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:21,912 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:21,912 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:21,912 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/B is initiating minor compaction (all files) 2024-11-24T11:17:21,912 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/A is initiating minor compaction (all files) 2024-11-24T11:17:21,912 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/A in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:21,912 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/B in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:21,912 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a968be65a33b41d5aa6c107f717c11dd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/308a89b48bd048e8b5a4ffddf4247897, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/3ea2c9b28791448ba9fec492699e145f] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=36.7 K 2024-11-24T11:17:21,912 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32059cb79721425791ceffa6621cef81, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aec62209bd7a44ba87d7362a755fd813, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/962b35c811dc4c95bc18f340ddd58599] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=92.3 K 2024-11-24T11:17:21,912 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:21,912 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. files: [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32059cb79721425791ceffa6621cef81, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aec62209bd7a44ba87d7362a755fd813, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/962b35c811dc4c95bc18f340ddd58599] 2024-11-24T11:17:21,912 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting a968be65a33b41d5aa6c107f717c11dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732447038265 2024-11-24T11:17:21,913 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32059cb79721425791ceffa6621cef81, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732447038265 2024-11-24T11:17:21,913 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 308a89b48bd048e8b5a4ffddf4247897, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732447038394 2024-11-24T11:17:21,913 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec62209bd7a44ba87d7362a755fd813, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732447038394 2024-11-24T11:17:21,913 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ea2c9b28791448ba9fec492699e145f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732447039052 2024-11-24T11:17:21,913 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962b35c811dc4c95bc18f340ddd58599, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732447039052 2024-11-24T11:17:21,919 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:21,921 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#B#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:21,921 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124b82c6e3479474219b548ac4cc2ddd3d1_fc46f0a8c7ecfcf606c3547de7a28906 store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:21,921 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/14907abda0f5447f8550bd5ab4c355fd is 50, key is test_row_0/B:col10/1732447039060/Put/seqid=0 2024-11-24T11:17:21,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742459_1635 (size=13119) 2024-11-24T11:17:21,925 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124b82c6e3479474219b548ac4cc2ddd3d1_fc46f0a8c7ecfcf606c3547de7a28906, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:21,925 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b82c6e3479474219b548ac4cc2ddd3d1_fc46f0a8c7ecfcf606c3547de7a28906 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:21,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742460_1636 (size=4469) 2024-11-24T11:17:21,929 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#A#compaction#532 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:21,929 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/e96ddcc59c074e83a972a6d9b243f7d7 is 175, key is test_row_0/A:col10/1732447039060/Put/seqid=0 2024-11-24T11:17:21,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742461_1637 (size=32073) 2024-11-24T11:17:22,335 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/14907abda0f5447f8550bd5ab4c355fd as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/14907abda0f5447f8550bd5ab4c355fd 2024-11-24T11:17:22,338 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/e96ddcc59c074e83a972a6d9b243f7d7 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/e96ddcc59c074e83a972a6d9b243f7d7 2024-11-24T11:17:22,340 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/B of fc46f0a8c7ecfcf606c3547de7a28906 into 14907abda0f5447f8550bd5ab4c355fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:22,340 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:22,340 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/B, priority=13, startTime=1732447041910; duration=0sec 2024-11-24T11:17:22,340 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T11:17:22,340 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:B 2024-11-24T11:17:22,340 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T11:17:22,341 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T11:17:22,341 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1541): fc46f0a8c7ecfcf606c3547de7a28906/C is initiating minor compaction (all files) 2024-11-24T11:17:22,341 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc46f0a8c7ecfcf606c3547de7a28906/C in TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:22,341 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e62a701eb95742edab4754996c8b92c7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/71c399ecc300400fb2c7254b595bc202, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/90cf3dcd385a4cd7be2f3856c59585e8] into tmpdir=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp, totalSize=36.7 K 2024-11-24T11:17:22,342 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting e62a701eb95742edab4754996c8b92c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732447038265 2024-11-24T11:17:22,342 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c399ecc300400fb2c7254b595bc202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732447038394 2024-11-24T11:17:22,342 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/A of fc46f0a8c7ecfcf606c3547de7a28906 into e96ddcc59c074e83a972a6d9b243f7d7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:22,342 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] compactions.Compactor(225): Compacting 90cf3dcd385a4cd7be2f3856c59585e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732447039052 2024-11-24T11:17:22,342 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:22,342 INFO [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/A, priority=13, startTime=1732447041910; duration=0sec 2024-11-24T11:17:22,342 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:22,342 DEBUG [RS:0;f5f73e76a839:45939-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:A 2024-11-24T11:17:22,349 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc46f0a8c7ecfcf606c3547de7a28906#C#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T11:17:22,349 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/58399bba95364315b15d48efe28740b1 is 50, key is test_row_0/C:col10/1732447039060/Put/seqid=0 2024-11-24T11:17:22,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742462_1638 (size=13119) 2024-11-24T11:17:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45939 {}] regionserver.HRegion(8855): Flush requested on fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:22,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T11:17:22,487 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-24T11:17:22,487 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-24T11:17:22,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:22,487 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:22,487 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:22,487 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:22,487 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:22,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:22,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:22,487 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:22,487 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:22,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:22,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:22,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:22,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:22,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T11:17:22,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a91f8db101164b41b6967a5a2d970930_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_0/A:col10/1732447040278/Put/seqid=0 2024-11-24T11:17:22,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742463_1639 (size=12454) 2024-11-24T11:17:22,501 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:22,504 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a91f8db101164b41b6967a5a2d970930_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a91f8db101164b41b6967a5a2d970930_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:22,505 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/805acbe03be443bc98eae2a4d5cefb02, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:22,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/805acbe03be443bc98eae2a4d5cefb02 is 175, key is test_row_0/A:col10/1732447040278/Put/seqid=0 2024-11-24T11:17:22,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742464_1640 (size=31255) 2024-11-24T11:17:22,755 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/58399bba95364315b15d48efe28740b1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/58399bba95364315b15d48efe28740b1 2024-11-24T11:17:22,759 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc46f0a8c7ecfcf606c3547de7a28906/C of fc46f0a8c7ecfcf606c3547de7a28906 into 58399bba95364315b15d48efe28740b1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T11:17:22,759 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:22,759 INFO [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., storeName=fc46f0a8c7ecfcf606c3547de7a28906/C, priority=13, startTime=1732447041911; duration=0sec 2024-11-24T11:17:22,759 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T11:17:22,759 DEBUG [RS:0;f5f73e76a839:45939-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc46f0a8c7ecfcf606c3547de7a28906:C 2024-11-24T11:17:22,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/805acbe03be443bc98eae2a4d5cefb02 2024-11-24T11:17:22,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/51e5964ae60b42b79af19bb7bac0b1f1 is 50, key is test_row_0/B:col10/1732447040278/Put/seqid=0 2024-11-24T11:17:22,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742465_1641 (size=12301) 2024-11-24T11:17:23,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/51e5964ae60b42b79af19bb7bac0b1f1 2024-11-24T11:17:23,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/6c1b6efe75084f168144b1ec85a1cbfb is 50, key is test_row_0/C:col10/1732447040278/Put/seqid=0 2024-11-24T11:17:23,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742466_1642 (size=12301) 2024-11-24T11:17:23,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/6c1b6efe75084f168144b1ec85a1cbfb 2024-11-24T11:17:23,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/805acbe03be443bc98eae2a4d5cefb02 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/805acbe03be443bc98eae2a4d5cefb02 2024-11-24T11:17:23,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/805acbe03be443bc98eae2a4d5cefb02, entries=150, sequenceid=376, filesize=30.5 K 2024-11-24T11:17:23,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/51e5964ae60b42b79af19bb7bac0b1f1 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/51e5964ae60b42b79af19bb7bac0b1f1 2024-11-24T11:17:23,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/51e5964ae60b42b79af19bb7bac0b1f1, entries=150, sequenceid=376, filesize=12.0 K 2024-11-24T11:17:23,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/6c1b6efe75084f168144b1ec85a1cbfb as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6c1b6efe75084f168144b1ec85a1cbfb 2024-11-24T11:17:23,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6c1b6efe75084f168144b1ec85a1cbfb, entries=150, sequenceid=376, filesize=12.0 K 2024-11-24T11:17:23,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=0 B/0 for fc46f0a8c7ecfcf606c3547de7a28906 in 1264ms, sequenceid=376, compaction requested=false 2024-11-24T11:17:23,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc46f0a8c7ecfcf606c3547de7a28906: 2024-11-24T11:17:26,135 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-24T11:17:26,135 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:26,135 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:26,135 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:26,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:26,155 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-24T11:17:26,155 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:26,155 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:26,155 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:26,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:28,865 INFO [Thread-2413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2413. 2024-11-24T11:17:28,865 DEBUG [Thread-2413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T11:17:28,865 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:28,865 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:28,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2796 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2852 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2779 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2798 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2848 2024-11-24T11:17:28,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T11:17:28,865 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:17:28,865 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:17:28,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:28,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:28,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:28,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-24T11:17:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T11:17:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T11:17:28,870 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732447048869"}]},"ts":"1732447048869"} 2024-11-24T11:17:28,871 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T11:17:28,871 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T11:17:28,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T11:17:28,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, UNASSIGN}] 2024-11-24T11:17:28,874 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, UNASSIGN 2024-11-24T11:17:28,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=CLOSING, regionLocation=f5f73e76a839,45939,1732446856176 2024-11-24T11:17:28,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, UNASSIGN because future has completed 2024-11-24T11:17:28,876 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T11:17:28,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176}] 2024-11-24T11:17:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T11:17:29,029 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:29,029 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T11:17:29,029 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing fc46f0a8c7ecfcf606c3547de7a28906, disabling compactions & flushes 2024-11-24T11:17:29,029 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:29,029 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. after waiting 0 ms 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:29,030 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing fc46f0a8c7ecfcf606c3547de7a28906 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=A 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=B 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc46f0a8c7ecfcf606c3547de7a28906, store=C 2024-11-24T11:17:29,030 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T11:17:29,039 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242541f87d396a4e1b80f5b24efbbf26bf_fc46f0a8c7ecfcf606c3547de7a28906 is 50, key is test_row_1/A:col10/1732447048863/Put/seqid=0 2024-11-24T11:17:29,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742467_1643 (size=9914) 2024-11-24T11:17:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T11:17:29,443 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T11:17:29,446 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242541f87d396a4e1b80f5b24efbbf26bf_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242541f87d396a4e1b80f5b24efbbf26bf_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:29,447 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1adbdc9204d14d9c99551e057c468744, store: [table=TestAcidGuarantees family=A region=fc46f0a8c7ecfcf606c3547de7a28906] 2024-11-24T11:17:29,448 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1adbdc9204d14d9c99551e057c468744 is 175, key is test_row_1/A:col10/1732447048863/Put/seqid=0 2024-11-24T11:17:29,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742468_1644 (size=22561) 2024-11-24T11:17:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T11:17:29,853 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1adbdc9204d14d9c99551e057c468744 2024-11-24T11:17:29,866 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/c2390592833b416f8c43910a4f375cb6 is 50, key is test_row_1/B:col10/1732447048863/Put/seqid=0 2024-11-24T11:17:29,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742469_1645 (size=9857) 2024-11-24T11:17:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T11:17:30,269 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/c2390592833b416f8c43910a4f375cb6 2024-11-24T11:17:30,275 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/5bb880621a514020909118963de4eaff is 50, key is test_row_1/C:col10/1732447048863/Put/seqid=0 2024-11-24T11:17:30,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742470_1646 (size=9857) 2024-11-24T11:17:30,678 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/5bb880621a514020909118963de4eaff 2024-11-24T11:17:30,683 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/A/1adbdc9204d14d9c99551e057c468744 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1adbdc9204d14d9c99551e057c468744 2024-11-24T11:17:30,687 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1adbdc9204d14d9c99551e057c468744, entries=100, sequenceid=383, filesize=22.0 K 2024-11-24T11:17:30,688 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/B/c2390592833b416f8c43910a4f375cb6 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/c2390592833b416f8c43910a4f375cb6 2024-11-24T11:17:30,691 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/c2390592833b416f8c43910a4f375cb6, entries=100, sequenceid=383, filesize=9.6 K 2024-11-24T11:17:30,692 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/.tmp/C/5bb880621a514020909118963de4eaff as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/5bb880621a514020909118963de4eaff 2024-11-24T11:17:30,694 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/5bb880621a514020909118963de4eaff, entries=100, sequenceid=383, filesize=9.6 K 2024-11-24T11:17:30,695 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fc46f0a8c7ecfcf606c3547de7a28906 in 1665ms, sequenceid=383, compaction requested=true 2024-11-24T11:17:30,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/55b3e7ae107c4dce850d79d8efa481fd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/c00775628d8b4b8dbb9a4e466670c8f4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7610805c5c54487aab7e22cbc809bdd3, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/667cb1ba955e403e8234f3baa9d36de2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/a3cc368a1ce24d708e41fb7ac3650ce9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/2d74a01255e44671881bd6c2f734db5b, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1f3f768774d34fc9b0a499ed39758360, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0991caa00191466ab59508b3cdf9efb7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/212c10e9648a4cd6a0ebb6a572ea7020, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aac5ee84601a4cedb26b2310d6c2efbe, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/704f11a3636f45bca42c678067c57254, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b6f4f977c5ef46e7ba22785df34921e0, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/f01bf9b24554475ab539961345c687cf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0ad5599cfea6489e8672dddab55f14a7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/5fd9e22b72744552825f2ce4bb1c5421, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32283add7ba64ff78249fe5f276e0fbf, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/d90d1411bdac4e389e9900698b451915, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7903488656104e18a7bf97d29c68bdc7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/8b5ca51989034cb4b2aaf1042081c01e, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/23e67b5c1ee449d48ffaa2b30d236ee4, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b01fcb2011864d72969c7cd311629a28, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32059cb79721425791ceffa6621cef81, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aec62209bd7a44ba87d7362a755fd813, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/962b35c811dc4c95bc18f340ddd58599] to archive 2024-11-24T11:17:30,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:17:30,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/55b3e7ae107c4dce850d79d8efa481fd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/55b3e7ae107c4dce850d79d8efa481fd 2024-11-24T11:17:30,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/c00775628d8b4b8dbb9a4e466670c8f4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/c00775628d8b4b8dbb9a4e466670c8f4 2024-11-24T11:17:30,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7610805c5c54487aab7e22cbc809bdd3 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7610805c5c54487aab7e22cbc809bdd3 2024-11-24T11:17:30,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/667cb1ba955e403e8234f3baa9d36de2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/667cb1ba955e403e8234f3baa9d36de2 2024-11-24T11:17:30,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/a3cc368a1ce24d708e41fb7ac3650ce9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/a3cc368a1ce24d708e41fb7ac3650ce9 2024-11-24T11:17:30,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/2d74a01255e44671881bd6c2f734db5b to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/2d74a01255e44671881bd6c2f734db5b 2024-11-24T11:17:30,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1f3f768774d34fc9b0a499ed39758360 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1f3f768774d34fc9b0a499ed39758360 2024-11-24T11:17:30,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0991caa00191466ab59508b3cdf9efb7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0991caa00191466ab59508b3cdf9efb7 2024-11-24T11:17:30,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/212c10e9648a4cd6a0ebb6a572ea7020 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/212c10e9648a4cd6a0ebb6a572ea7020 2024-11-24T11:17:30,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aac5ee84601a4cedb26b2310d6c2efbe to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aac5ee84601a4cedb26b2310d6c2efbe 2024-11-24T11:17:30,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/704f11a3636f45bca42c678067c57254 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/704f11a3636f45bca42c678067c57254 2024-11-24T11:17:30,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b6f4f977c5ef46e7ba22785df34921e0 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b6f4f977c5ef46e7ba22785df34921e0 2024-11-24T11:17:30,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/f01bf9b24554475ab539961345c687cf to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/f01bf9b24554475ab539961345c687cf 2024-11-24T11:17:30,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0ad5599cfea6489e8672dddab55f14a7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/0ad5599cfea6489e8672dddab55f14a7 2024-11-24T11:17:30,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/5fd9e22b72744552825f2ce4bb1c5421 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/5fd9e22b72744552825f2ce4bb1c5421 2024-11-24T11:17:30,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32283add7ba64ff78249fe5f276e0fbf to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32283add7ba64ff78249fe5f276e0fbf 2024-11-24T11:17:30,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/d90d1411bdac4e389e9900698b451915 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/d90d1411bdac4e389e9900698b451915 2024-11-24T11:17:30,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7903488656104e18a7bf97d29c68bdc7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/7903488656104e18a7bf97d29c68bdc7 2024-11-24T11:17:30,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/8b5ca51989034cb4b2aaf1042081c01e to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/8b5ca51989034cb4b2aaf1042081c01e 2024-11-24T11:17:30,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/23e67b5c1ee449d48ffaa2b30d236ee4 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/23e67b5c1ee449d48ffaa2b30d236ee4 2024-11-24T11:17:30,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b01fcb2011864d72969c7cd311629a28 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/b01fcb2011864d72969c7cd311629a28 2024-11-24T11:17:30,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32059cb79721425791ceffa6621cef81 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/32059cb79721425791ceffa6621cef81 2024-11-24T11:17:30,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aec62209bd7a44ba87d7362a755fd813 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/aec62209bd7a44ba87d7362a755fd813 2024-11-24T11:17:30,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/962b35c811dc4c95bc18f340ddd58599 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/962b35c811dc4c95bc18f340ddd58599 2024-11-24T11:17:30,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/9a7cc0a190e940f9bb01a3161d3bfcdb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a828e5b4da394adca481354831b9ce8d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/7ecf7a4e58c44f96ad1181e191194b68, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fda36dc21903479b8b94006dc1b7195f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0dc2f0f2b7b43caa47f0d17c686f1dd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/60cbafbe1a064420ab44bd1da472b927, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fd4533e65bc04066947b45c98514cce6, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/1955c3ed1e9a406daa9082660ac35ac5, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/f5133d7c9fb0467fbc768a0f5de6d880, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/8c403502b670400f900cc2ef00422e90, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/719e69c7ecd846b795de9f1df0a6ef40, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/6e0f3eae44e24ab195e1291f855a546c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0ad62fc2f7747db84f8d04f0a65f109, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/73b2f990a87640cf92dbc27e4cbdc2a9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a76e53335b184233b2ce3255199fafdb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/491f70eae9074434bcc802e4ba04d306, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b6e516ee2888424fa8df1683f6b35e8a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/afeeb3200c0949b99219e0be99824ca1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/67ea8e240f114c8d8685bb95d23a9252, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/e657f6521ed340e6b4d3622c175f9ebb, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a968be65a33b41d5aa6c107f717c11dd, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/61256eda4ce446ab86c33a50e516c1e9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/308a89b48bd048e8b5a4ffddf4247897, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/3ea2c9b28791448ba9fec492699e145f] to archive 2024-11-24T11:17:30,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:17:30,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/9a7cc0a190e940f9bb01a3161d3bfcdb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/9a7cc0a190e940f9bb01a3161d3bfcdb 2024-11-24T11:17:30,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a828e5b4da394adca481354831b9ce8d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a828e5b4da394adca481354831b9ce8d 2024-11-24T11:17:30,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/7ecf7a4e58c44f96ad1181e191194b68 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/7ecf7a4e58c44f96ad1181e191194b68 2024-11-24T11:17:30,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fda36dc21903479b8b94006dc1b7195f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fda36dc21903479b8b94006dc1b7195f 2024-11-24T11:17:30,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0dc2f0f2b7b43caa47f0d17c686f1dd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0dc2f0f2b7b43caa47f0d17c686f1dd 2024-11-24T11:17:30,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/60cbafbe1a064420ab44bd1da472b927 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/60cbafbe1a064420ab44bd1da472b927 2024-11-24T11:17:30,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fd4533e65bc04066947b45c98514cce6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/fd4533e65bc04066947b45c98514cce6 2024-11-24T11:17:30,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/1955c3ed1e9a406daa9082660ac35ac5 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/1955c3ed1e9a406daa9082660ac35ac5 2024-11-24T11:17:30,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/f5133d7c9fb0467fbc768a0f5de6d880 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/f5133d7c9fb0467fbc768a0f5de6d880 2024-11-24T11:17:30,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/8c403502b670400f900cc2ef00422e90 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/8c403502b670400f900cc2ef00422e90 2024-11-24T11:17:30,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/719e69c7ecd846b795de9f1df0a6ef40 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/719e69c7ecd846b795de9f1df0a6ef40 2024-11-24T11:17:30,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/6e0f3eae44e24ab195e1291f855a546c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/6e0f3eae44e24ab195e1291f855a546c 2024-11-24T11:17:30,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0ad62fc2f7747db84f8d04f0a65f109 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b0ad62fc2f7747db84f8d04f0a65f109 2024-11-24T11:17:30,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/73b2f990a87640cf92dbc27e4cbdc2a9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/73b2f990a87640cf92dbc27e4cbdc2a9 2024-11-24T11:17:30,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a76e53335b184233b2ce3255199fafdb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a76e53335b184233b2ce3255199fafdb 2024-11-24T11:17:30,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/491f70eae9074434bcc802e4ba04d306 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/491f70eae9074434bcc802e4ba04d306 2024-11-24T11:17:30,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b6e516ee2888424fa8df1683f6b35e8a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/b6e516ee2888424fa8df1683f6b35e8a 2024-11-24T11:17:30,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/afeeb3200c0949b99219e0be99824ca1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/afeeb3200c0949b99219e0be99824ca1 2024-11-24T11:17:30,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/67ea8e240f114c8d8685bb95d23a9252 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/67ea8e240f114c8d8685bb95d23a9252 2024-11-24T11:17:30,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/e657f6521ed340e6b4d3622c175f9ebb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/e657f6521ed340e6b4d3622c175f9ebb 2024-11-24T11:17:30,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a968be65a33b41d5aa6c107f717c11dd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/a968be65a33b41d5aa6c107f717c11dd 2024-11-24T11:17:30,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/61256eda4ce446ab86c33a50e516c1e9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/61256eda4ce446ab86c33a50e516c1e9 2024-11-24T11:17:30,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/308a89b48bd048e8b5a4ffddf4247897 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/308a89b48bd048e8b5a4ffddf4247897 2024-11-24T11:17:30,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/3ea2c9b28791448ba9fec492699e145f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/3ea2c9b28791448ba9fec492699e145f 2024-11-24T11:17:30,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/168867cb7ab5455487bb5f197aeb31fa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/07e02b19bbc5451a94cc4da4f5a28653, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/d6ecd803a77b4a4ebb2cd7da14b3a8fa, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/31faf07306f742b0ac380fb29211f8fc, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/82680ef026944db6b190d900fe65c58a, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b7a1daa1d74442f5800b3723835f112d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/57b2bd151c9b4ea4bc582f802436a9dc, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/af3d001c8d21457ea6dcb34a8582fd32, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6ab52cc006da4546bc802ba1b09baf76, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/003fb6aeccb549fb84dd70033995ffd2, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c9fe447872f4420792e63ee69921cc52, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/1d1d936d3cc340db9de120daee44cf70, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/205da77fccec410e8397cb729d2d8d35, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c7003f30b8fe458581969263f92b2e2f, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/526840c295e84dd7bf4d3295e712852d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/ce486dbd319c48d3a18be81ba00ba136, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b6c15997ba3d4a7aad404eabe844046d, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/26ef15ce85244463bf3724f1c6399fc1, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/54698cfac4b24d3381779afcba0e581c, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e17ee16f543342c099ea068ced8b7cf9, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e62a701eb95742edab4754996c8b92c7, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b3f9908b8cbc4632a518095d27506174, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/71c399ecc300400fb2c7254b595bc202, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/90cf3dcd385a4cd7be2f3856c59585e8] to archive 2024-11-24T11:17:30,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T11:17:30,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/168867cb7ab5455487bb5f197aeb31fa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/168867cb7ab5455487bb5f197aeb31fa 2024-11-24T11:17:30,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/07e02b19bbc5451a94cc4da4f5a28653 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/07e02b19bbc5451a94cc4da4f5a28653 2024-11-24T11:17:30,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/d6ecd803a77b4a4ebb2cd7da14b3a8fa to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/d6ecd803a77b4a4ebb2cd7da14b3a8fa 2024-11-24T11:17:30,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/31faf07306f742b0ac380fb29211f8fc to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/31faf07306f742b0ac380fb29211f8fc 2024-11-24T11:17:30,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/82680ef026944db6b190d900fe65c58a to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/82680ef026944db6b190d900fe65c58a 2024-11-24T11:17:30,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b7a1daa1d74442f5800b3723835f112d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b7a1daa1d74442f5800b3723835f112d 2024-11-24T11:17:30,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/57b2bd151c9b4ea4bc582f802436a9dc to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/57b2bd151c9b4ea4bc582f802436a9dc 2024-11-24T11:17:30,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/af3d001c8d21457ea6dcb34a8582fd32 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/af3d001c8d21457ea6dcb34a8582fd32 2024-11-24T11:17:30,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6ab52cc006da4546bc802ba1b09baf76 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6ab52cc006da4546bc802ba1b09baf76 2024-11-24T11:17:30,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/003fb6aeccb549fb84dd70033995ffd2 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/003fb6aeccb549fb84dd70033995ffd2 2024-11-24T11:17:30,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c9fe447872f4420792e63ee69921cc52 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c9fe447872f4420792e63ee69921cc52 2024-11-24T11:17:30,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/1d1d936d3cc340db9de120daee44cf70 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/1d1d936d3cc340db9de120daee44cf70 2024-11-24T11:17:30,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/205da77fccec410e8397cb729d2d8d35 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/205da77fccec410e8397cb729d2d8d35 2024-11-24T11:17:30,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c7003f30b8fe458581969263f92b2e2f to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/c7003f30b8fe458581969263f92b2e2f 2024-11-24T11:17:30,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/526840c295e84dd7bf4d3295e712852d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/526840c295e84dd7bf4d3295e712852d 2024-11-24T11:17:30,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/ce486dbd319c48d3a18be81ba00ba136 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/ce486dbd319c48d3a18be81ba00ba136 2024-11-24T11:17:30,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b6c15997ba3d4a7aad404eabe844046d to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b6c15997ba3d4a7aad404eabe844046d 2024-11-24T11:17:30,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/26ef15ce85244463bf3724f1c6399fc1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/26ef15ce85244463bf3724f1c6399fc1 2024-11-24T11:17:30,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/54698cfac4b24d3381779afcba0e581c to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/54698cfac4b24d3381779afcba0e581c 2024-11-24T11:17:30,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e17ee16f543342c099ea068ced8b7cf9 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e17ee16f543342c099ea068ced8b7cf9 2024-11-24T11:17:30,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e62a701eb95742edab4754996c8b92c7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/e62a701eb95742edab4754996c8b92c7 2024-11-24T11:17:30,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b3f9908b8cbc4632a518095d27506174 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/b3f9908b8cbc4632a518095d27506174 2024-11-24T11:17:30,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/71c399ecc300400fb2c7254b595bc202 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/71c399ecc300400fb2c7254b595bc202 2024-11-24T11:17:30,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/90cf3dcd385a4cd7be2f3856c59585e8 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/90cf3dcd385a4cd7be2f3856c59585e8 2024-11-24T11:17:30,764 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/recovered.edits/386.seqid, newMaxSeqId=386, maxSeqId=4 2024-11-24T11:17:30,764 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. 2024-11-24T11:17:30,764 DEBUG [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for fc46f0a8c7ecfcf606c3547de7a28906: Waiting for close lock at 1732447049029Running coprocessor pre-close hooks at 1732447049029Disabling compacts and flushes for region at 1732447049029Disabling writes for close at 1732447049030 (+1 ms)Obtaining lock to block concurrent updates at 1732447049030Preparing flush snapshotting stores in fc46f0a8c7ecfcf606c3547de7a28906 at 1732447049030Finished memstore snapshotting TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732447049030Flushing stores of TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906. at 1732447049031 (+1 ms)Flushing fc46f0a8c7ecfcf606c3547de7a28906/A: creating writer at 1732447049031Flushing mob file fc46f0a8c7ecfcf606c3547de7a28906/A: appending metadata at 1732447049039 (+8 ms)Flushing mob file fc46f0a8c7ecfcf606c3547de7a28906/A: closing flushed file at 1732447049039Flushing fc46f0a8c7ecfcf606c3547de7a28906/A: appending metadata at 1732447049447 (+408 ms)Flushing fc46f0a8c7ecfcf606c3547de7a28906/A: closing flushed file at 1732447049447Flushing fc46f0a8c7ecfcf606c3547de7a28906/B: creating writer at 1732447049863 (+416 ms)Flushing fc46f0a8c7ecfcf606c3547de7a28906/B: appending metadata at 1732447049866 (+3 ms)Flushing fc46f0a8c7ecfcf606c3547de7a28906/B: closing flushed file at 1732447049866Flushing fc46f0a8c7ecfcf606c3547de7a28906/C: creating writer at 1732447050272 (+406 ms)Flushing fc46f0a8c7ecfcf606c3547de7a28906/C: appending metadata at 1732447050275 (+3 ms)Flushing fc46f0a8c7ecfcf606c3547de7a28906/C: closing flushed file at 1732447050275Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30f2007d: reopening flushed file at 1732447050683 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2939d291: reopening flushed file at 1732447050687 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5014836c: reopening flushed file at 1732447050691 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fc46f0a8c7ecfcf606c3547de7a28906 in 1665ms, sequenceid=383, compaction requested=true at 1732447050695 (+4 ms)Writing region close event to WAL at 1732447050762 (+67 ms)Running coprocessor post-close hooks at 1732447050764 (+2 ms)Closed at 1732447050764 2024-11-24T11:17:30,766 INFO [RS_CLOSE_REGION-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:30,767 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=fc46f0a8c7ecfcf606c3547de7a28906, regionState=CLOSED 2024-11-24T11:17:30,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 because future has completed 2024-11-24T11:17:30,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-24T11:17:30,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure fc46f0a8c7ecfcf606c3547de7a28906, server=f5f73e76a839,45939,1732446856176 in 1.8930 sec 2024-11-24T11:17:30,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-24T11:17:30,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc46f0a8c7ecfcf606c3547de7a28906, UNASSIGN in 1.8970 sec 2024-11-24T11:17:30,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-24T11:17:30,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9000 sec 2024-11-24T11:17:30,775 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732447050775"}]},"ts":"1732447050775"} 2024-11-24T11:17:30,776 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T11:17:30,776 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T11:17:30,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-24T11:17:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T11:17:31,014 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:31,015 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-24T11:17:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:17:31,018 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:17:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T11:17:31,019 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:17:31,024 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,027 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C, FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/recovered.edits] 2024-11-24T11:17:31,031 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1adbdc9204d14d9c99551e057c468744 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/1adbdc9204d14d9c99551e057c468744 2024-11-24T11:17:31,032 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/805acbe03be443bc98eae2a4d5cefb02 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/805acbe03be443bc98eae2a4d5cefb02 2024-11-24T11:17:31,034 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/e96ddcc59c074e83a972a6d9b243f7d7 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/A/e96ddcc59c074e83a972a6d9b243f7d7 2024-11-24T11:17:31,035 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/14907abda0f5447f8550bd5ab4c355fd to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/14907abda0f5447f8550bd5ab4c355fd 2024-11-24T11:17:31,036 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/51e5964ae60b42b79af19bb7bac0b1f1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/51e5964ae60b42b79af19bb7bac0b1f1 2024-11-24T11:17:31,037 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/c2390592833b416f8c43910a4f375cb6 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/B/c2390592833b416f8c43910a4f375cb6 2024-11-24T11:17:31,039 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/58399bba95364315b15d48efe28740b1 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/58399bba95364315b15d48efe28740b1 2024-11-24T11:17:31,040 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/5bb880621a514020909118963de4eaff to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/5bb880621a514020909118963de4eaff 2024-11-24T11:17:31,041 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6c1b6efe75084f168144b1ec85a1cbfb to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/C/6c1b6efe75084f168144b1ec85a1cbfb 2024-11-24T11:17:31,043 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/recovered.edits/386.seqid to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906/recovered.edits/386.seqid 2024-11-24T11:17:31,043 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/default/TestAcidGuarantees/fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,043 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T11:17:31,044 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T11:17:31,045 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T11:17:31,048 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124056db37c62254c3ea40f5e9f84a6cd67_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124056db37c62254c3ea40f5e9f84a6cd67_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,050 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124240a43a624504f0980856bd4405958db_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124240a43a624504f0980856bd4405958db_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,051 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242541f87d396a4e1b80f5b24efbbf26bf_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242541f87d396a4e1b80f5b24efbbf26bf_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,052 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243a1e17d34797441a9fbb0b84526e29f1_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243a1e17d34797441a9fbb0b84526e29f1_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,053 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112452317ccf668343eaa2687f718264776f_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112452317ccf668343eaa2687f718264776f_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,055 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124547c701f50a148b1bef442766c80b785_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124547c701f50a148b1bef442766c80b785_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,056 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456e1136213a943b680cafa477b5929ae_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456e1136213a943b680cafa477b5929ae_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,058 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112461a5a453d0194738b5f51d27875a0e42_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112461a5a453d0194738b5f51d27875a0e42_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,059 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246f0256bcd2b0436696aa16a274a0bcb8_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246f0256bcd2b0436696aa16a274a0bcb8_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,061 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a91f8db101164b41b6967a5a2d970930_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a91f8db101164b41b6967a5a2d970930_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,062 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b2f88bad6a4a42b58f08d84ef1e28f14_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b2f88bad6a4a42b58f08d84ef1e28f14_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,064 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b39883457e834554a2551b5026d60823_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b39883457e834554a2551b5026d60823_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,066 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c1473e565bc94ef08bab259ff602cedd_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c1473e565bc94ef08bab259ff602cedd_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,067 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c730bb26e2be45f18fe02508c84e7301_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c730bb26e2be45f18fe02508c84e7301_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,069 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ccdf5841d0944683b1dfe5e51152a922_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ccdf5841d0944683b1dfe5e51152a922_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,071 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d531d3e71204474bbbe85cbf0564459e_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d531d3e71204474bbbe85cbf0564459e_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,072 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dd1b90e393ca4bad85c2f288e648a3f8_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dd1b90e393ca4bad85c2f288e648a3f8_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,074 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e1254ed974494495b752b83eb22d27cb_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e1254ed974494495b752b83eb22d27cb_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,075 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f0f551a05cc347b285d13eebdd395c06_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f0f551a05cc347b285d13eebdd395c06_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,077 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f91d1c39db494046876721c27cc57d95_fc46f0a8c7ecfcf606c3547de7a28906 to hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f91d1c39db494046876721c27cc57d95_fc46f0a8c7ecfcf606c3547de7a28906 2024-11-24T11:17:31,078 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T11:17:31,081 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:17:31,084 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T11:17:31,087 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T11:17:31,089 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:17:31,089 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T11:17:31,089 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732447051089"}]},"ts":"9223372036854775807"} 2024-11-24T11:17:31,092 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T11:17:31,092 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fc46f0a8c7ecfcf606c3547de7a28906, NAME => 'TestAcidGuarantees,,1732447017160.fc46f0a8c7ecfcf606c3547de7a28906.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T11:17:31,092 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T11:17:31,092 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732447051092"}]},"ts":"9223372036854775807"} 2024-11-24T11:17:31,095 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T11:17:31,096 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T11:17:31,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-24T11:17:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40345 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T11:17:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T11:17:31,128 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T11:17:31,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 246) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=394 (was 355) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6504 (was 5923) - AvailableMemoryMB LEAK? - 2024-11-24T11:17:31,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T11:17:31,140 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:17:31,140 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:17:31,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:31,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:31,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T11:17:31,140 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T11:17:31,140 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-24T11:17:31,141 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5f73e76a839,40345,1732446855400 2024-11-24T11:17:31,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T11:17:31,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T11:17:31,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:17:31,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:17:31,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T11:17:31,194 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T11:17:31,194 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:17:31,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T11:17:31,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:31,195 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T11:17:31,195 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5f73e76a839,45939,1732446856176' ***** 2024-11-24T11:17:31,195 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-24T11:17:31,195 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T11:17:31,197 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T11:17:31,197 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T11:17:31,197 INFO [RS:0;f5f73e76a839:45939 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(959): stopping server f5f73e76a839,45939,1732446856176 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5f73e76a839:45939. 2024-11-24T11:17:31,198 DEBUG [RS:0;f5f73e76a839:45939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T11:17:31,198 DEBUG [RS:0;f5f73e76a839:45939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T11:17:31,198 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T11:17:31,199 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T11:17:31,199 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T11:17:31,200 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:31,200 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T11:17:31,200 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T11:17:31,200 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T11:17:31,200 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T11:17:31,200 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T11:17:31,200 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-24T11:17:31,220 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/info/d24549382207408a85f55a047624656c is 95, key is TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c./info:/1732446887233/DeleteFamily/seqid=0 2024-11-24T11:17:31,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742471_1647 (size=6514) 2024-11-24T11:17:31,276 INFO [regionserver/f5f73e76a839:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T11:17:31,400 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:31,600 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:31,624 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/info/d24549382207408a85f55a047624656c 2024-11-24T11:17:31,646 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/ns/367c8f33a7b14c73a97975a04495f17b is 93, key is TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c./ns:/1732446887187/DeleteFamily/seqid=0 2024-11-24T11:17:31,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742472_1648 (size=5906) 2024-11-24T11:17:31,800 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:32,001 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:32,049 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/ns/367c8f33a7b14c73a97975a04495f17b 2024-11-24T11:17:32,067 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/rep_barrier/9940040487bc4590a68f57a974e0582b is 102, key is TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c./rep_barrier:/1732446887187/DeleteFamily/seqid=0 2024-11-24T11:17:32,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742473_1649 (size=6025) 2024-11-24T11:17:32,161 INFO [regionserver/f5f73e76a839:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T11:17:32,161 INFO [regionserver/f5f73e76a839:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T11:17:32,201 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:32,401 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:32,471 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/rep_barrier/9940040487bc4590a68f57a974e0582b 2024-11-24T11:17:32,487 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/table/5f5bc3417c40480081cf2820c93ae8d3 is 96, key is TestAcidGuarantees,,1732446860764.358290acb08925e00ab2bb913fcaac3c./table:/1732446887187/DeleteFamily/seqid=0 2024-11-24T11:17:32,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742474_1650 (size=5936) 2024-11-24T11:17:32,601 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:32,801 DEBUG [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T11:17:32,890 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/table/5f5bc3417c40480081cf2820c93ae8d3 2024-11-24T11:17:32,894 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/info/d24549382207408a85f55a047624656c as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/info/d24549382207408a85f55a047624656c 2024-11-24T11:17:32,896 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/info/d24549382207408a85f55a047624656c, entries=12, sequenceid=90, filesize=6.4 K 2024-11-24T11:17:32,897 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/ns/367c8f33a7b14c73a97975a04495f17b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/ns/367c8f33a7b14c73a97975a04495f17b 2024-11-24T11:17:32,901 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/ns/367c8f33a7b14c73a97975a04495f17b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-24T11:17:32,902 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/rep_barrier/9940040487bc4590a68f57a974e0582b as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/rep_barrier/9940040487bc4590a68f57a974e0582b 2024-11-24T11:17:32,906 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/rep_barrier/9940040487bc4590a68f57a974e0582b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-24T11:17:32,906 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/.tmp/table/5f5bc3417c40480081cf2820c93ae8d3 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/table/5f5bc3417c40480081cf2820c93ae8d3 2024-11-24T11:17:32,909 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/table/5f5bc3417c40480081cf2820c93ae8d3, entries=7, sequenceid=90, filesize=5.8 K 2024-11-24T11:17:32,910 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false 2024-11-24T11:17:32,918 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-24T11:17:32,919 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T11:17:32,919 INFO [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T11:17:32,919 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732447051199Running coprocessor pre-close hooks at 1732447051200 (+1 ms)Disabling compacts and flushes for region at 1732447051200Disabling writes for close at 1732447051200Obtaining lock to block concurrent updates at 1732447051200Preparing flush snapshotting stores in 1588230740 at 1732447051200Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732447051201 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732447051201Flushing 1588230740/info: creating writer at 1732447051202 (+1 ms)Flushing 1588230740/info: appending metadata at 1732447051220 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732447051220Flushing 1588230740/ns: creating writer at 1732447051629 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732447051646 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732447051646Flushing 1588230740/rep_barrier: creating writer at 1732447052053 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732447052066 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732447052066Flushing 1588230740/table: creating writer at 1732447052473 (+407 ms)Flushing 1588230740/table: appending metadata at 1732447052487 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732447052487Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@925805d: reopening flushed file at 1732447052893 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7637bc2e: reopening flushed file at 1732447052897 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52af5538: reopening flushed file at 1732447052902 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e8f0367: reopening flushed file at 1732447052906 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false at 1732447052910 (+4 ms)Writing region close event to WAL at 1732447052916 (+6 ms)Running coprocessor post-close hooks at 1732447052919 (+3 ms)Closed at 1732447052919 2024-11-24T11:17:32,919 DEBUG [RS_CLOSE_META-regionserver/f5f73e76a839:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T11:17:33,002 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(976): stopping server f5f73e76a839,45939,1732446856176; all regions closed. 2024-11-24T11:17:33,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741834_1010 (size=25344) 2024-11-24T11:17:33,009 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f5f73e76a839:40345 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-24T11:17:33,011 DEBUG [RS:0;f5f73e76a839:45939 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/oldWALs 2024-11-24T11:17:33,011 INFO [RS:0;f5f73e76a839:45939 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f5f73e76a839%2C45939%2C1732446856176.meta:.meta(num 1732446860362) 2024-11-24T11:17:33,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741832_1008 (size=15809524) 2024-11-24T11:17:33,015 DEBUG [RS:0;f5f73e76a839:45939 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/oldWALs 2024-11-24T11:17:33,015 INFO [RS:0;f5f73e76a839:45939 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f5f73e76a839%2C45939%2C1732446856176:(num 1732446859312) 2024-11-24T11:17:33,015 DEBUG [RS:0;f5f73e76a839:45939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T11:17:33,015 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T11:17:33,015 INFO [RS:0;f5f73e76a839:45939 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T11:17:33,015 INFO [RS:0;f5f73e76a839:45939 {}] hbase.ChoreService(370): Chore service for: regionserver/f5f73e76a839:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T11:17:33,016 INFO [RS:0;f5f73e76a839:45939 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T11:17:33,016 INFO [regionserver/f5f73e76a839:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T11:17:33,017 INFO [RS:0;f5f73e76a839:45939 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45939 2024-11-24T11:17:33,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5f73e76a839,45939,1732446856176 2024-11-24T11:17:33,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T11:17:33,032 INFO [RS:0;f5f73e76a839:45939 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T11:17:33,044 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5f73e76a839,45939,1732446856176] 2024-11-24T11:17:33,054 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5f73e76a839,45939,1732446856176 already deleted, retry=false 2024-11-24T11:17:33,054 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5f73e76a839,45939,1732446856176 expired; onlineServers=0 2024-11-24T11:17:33,055 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5f73e76a839,40345,1732446855400' ***** 2024-11-24T11:17:33,055 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T11:17:33,055 INFO [M:0;f5f73e76a839:40345 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T11:17:33,055 INFO [M:0;f5f73e76a839:40345 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T11:17:33,055 DEBUG [M:0;f5f73e76a839:40345 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T11:17:33,055 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T11:17:33,055 DEBUG [M:0;f5f73e76a839:40345 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T11:17:33,055 DEBUG [master/f5f73e76a839:0:becomeActiveMaster-HFileCleaner.small.0-1732446859024 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5f73e76a839:0:becomeActiveMaster-HFileCleaner.small.0-1732446859024,5,FailOnTimeoutGroup] 2024-11-24T11:17:33,055 DEBUG [master/f5f73e76a839:0:becomeActiveMaster-HFileCleaner.large.0-1732446859021 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5f73e76a839:0:becomeActiveMaster-HFileCleaner.large.0-1732446859021,5,FailOnTimeoutGroup] 2024-11-24T11:17:33,056 INFO [M:0;f5f73e76a839:40345 {}] hbase.ChoreService(370): Chore service for: master/f5f73e76a839:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T11:17:33,056 INFO [M:0;f5f73e76a839:40345 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T11:17:33,056 DEBUG [M:0;f5f73e76a839:40345 {}] master.HMaster(1795): Stopping service threads 2024-11-24T11:17:33,056 INFO [M:0;f5f73e76a839:40345 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T11:17:33,056 INFO [M:0;f5f73e76a839:40345 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T11:17:33,056 ERROR [M:0;f5f73e76a839:40345 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1420519831) connection to localhost/127.0.0.1:41675 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41675,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-24T11:17:33,057 INFO [M:0;f5f73e76a839:40345 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T11:17:33,057 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T11:17:33,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T11:17:33,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T11:17:33,065 DEBUG [M:0;f5f73e76a839:40345 {}] zookeeper.ZKUtil(347): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T11:17:33,065 WARN [M:0;f5f73e76a839:40345 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T11:17:33,066 INFO [M:0;f5f73e76a839:40345 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/.lastflushedseqids 2024-11-24T11:17:33,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742475_1651 (size=72) 2024-11-24T11:17:33,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T11:17:33,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45939-0x1016c8d19510001, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T11:17:33,145 INFO [RS:0;f5f73e76a839:45939 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T11:17:33,145 INFO [RS:0;f5f73e76a839:45939 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5f73e76a839,45939,1732446856176; zookeeper connection closed. 2024-11-24T11:17:33,145 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@34c5f1fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@34c5f1fa 2024-11-24T11:17:33,146 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T11:17:33,480 INFO [M:0;f5f73e76a839:40345 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T11:17:33,480 INFO [M:0;f5f73e76a839:40345 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T11:17:33,481 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T11:17:33,481 INFO [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:17:33,481 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:17:33,481 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T11:17:33,481 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:17:33,481 INFO [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=784.06 KB heapSize=961.80 KB 2024-11-24T11:17:33,501 DEBUG [M:0;f5f73e76a839:40345 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/66c7d5bc35dc45efab61483a4287d8ff is 82, key is hbase:meta,,1/info:regioninfo/1732446860447/Put/seqid=0 2024-11-24T11:17:33,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742476_1652 (size=5672) 2024-11-24T11:17:33,905 INFO [M:0;f5f73e76a839:40345 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2200 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/66c7d5bc35dc45efab61483a4287d8ff 2024-11-24T11:17:33,931 DEBUG [M:0;f5f73e76a839:40345 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0bb769bad1654a7e8d49a17ba079c4b5 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1732447020310/Put/seqid=0 2024-11-24T11:17:33,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742477_1653 (size=44651) 2024-11-24T11:17:34,334 INFO [M:0;f5f73e76a839:40345 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=783.50 KB at sequenceid=2200 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0bb769bad1654a7e8d49a17ba079c4b5 2024-11-24T11:17:34,338 INFO [M:0;f5f73e76a839:40345 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0bb769bad1654a7e8d49a17ba079c4b5 2024-11-24T11:17:34,354 DEBUG [M:0;f5f73e76a839:40345 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c0e5e55b47544195920709e9d6009477 is 69, key is f5f73e76a839,45939,1732446856176/rs:state/1732446859071/Put/seqid=0 2024-11-24T11:17:34,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073742478_1654 (size=5156) 2024-11-24T11:17:34,758 INFO [M:0;f5f73e76a839:40345 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2200 (bloomFilter=true), to=hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c0e5e55b47544195920709e9d6009477 2024-11-24T11:17:34,762 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/66c7d5bc35dc45efab61483a4287d8ff as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/66c7d5bc35dc45efab61483a4287d8ff 2024-11-24T11:17:34,766 INFO [M:0;f5f73e76a839:40345 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/66c7d5bc35dc45efab61483a4287d8ff, entries=8, sequenceid=2200, filesize=5.5 K 2024-11-24T11:17:34,767 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0bb769bad1654a7e8d49a17ba079c4b5 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0bb769bad1654a7e8d49a17ba079c4b5 2024-11-24T11:17:34,771 INFO [M:0;f5f73e76a839:40345 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0bb769bad1654a7e8d49a17ba079c4b5 2024-11-24T11:17:34,771 INFO [M:0;f5f73e76a839:40345 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0bb769bad1654a7e8d49a17ba079c4b5, entries=166, sequenceid=2200, filesize=43.6 K 2024-11-24T11:17:34,772 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c0e5e55b47544195920709e9d6009477 as hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c0e5e55b47544195920709e9d6009477 2024-11-24T11:17:34,774 INFO [M:0;f5f73e76a839:40345 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41675/user/jenkins/test-data/1ac9cae5-21d5-ecf3-f195-d202179c8e6c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c0e5e55b47544195920709e9d6009477, entries=1, sequenceid=2200, filesize=5.0 K 2024-11-24T11:17:34,775 INFO [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(3140): Finished flush of dataSize ~784.06 KB/802873, heapSize ~961.50 KB/984576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=2200, compaction requested=false 2024-11-24T11:17:34,776 INFO [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T11:17:34,776 DEBUG [M:0;f5f73e76a839:40345 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732447053480Disabling compacts and flushes for region at 1732447053480Disabling writes for close at 1732447053481 (+1 ms)Obtaining lock to block concurrent updates at 1732447053481Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732447053481Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=802873, getHeapSize=984816, getOffHeapSize=0, getCellsCount=2391 at 1732447053482 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732447053483 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732447053483Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732447053501 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732447053501Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732447053910 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732447053930 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732447053930Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732447054338 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732447054354 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732447054354Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6241d507: reopening flushed file at 1732447054761 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43cabcc5: reopening flushed file at 1732447054766 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd37a35: reopening flushed file at 1732447054771 (+5 ms)Finished flush of dataSize ~784.06 KB/802873, heapSize ~961.50 KB/984576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=2200, compaction requested=false at 1732447054775 (+4 ms)Writing region close event to WAL at 1732447054776 (+1 ms)Closed at 1732447054776 2024-11-24T11:17:34,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34189 is added to blk_1073741830_1006 (size=947056) 2024-11-24T11:17:34,779 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T11:17:34,779 INFO [M:0;f5f73e76a839:40345 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T11:17:34,779 INFO [M:0;f5f73e76a839:40345 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40345 2024-11-24T11:17:34,780 INFO [M:0;f5f73e76a839:40345 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T11:17:34,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T11:17:34,923 INFO [M:0;f5f73e76a839:40345 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T11:17:34,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40345-0x1016c8d19510000, quorum=127.0.0.1:57086, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T11:17:34,933 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T11:17:34,936 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T11:17:34,936 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T11:17:34,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T11:17:34,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/hadoop.log.dir/,STOPPED} 2024-11-24T11:17:34,940 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T11:17:34,940 WARN [BP-1825513930-172.17.0.3-1732446851790 heartbeating to localhost/127.0.0.1:41675 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T11:17:34,940 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T11:17:34,940 WARN [BP-1825513930-172.17.0.3-1732446851790 heartbeating to localhost/127.0.0.1:41675 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1825513930-172.17.0.3-1732446851790 (Datanode Uuid 2f4fbaac-9b8c-4e23-b847-b1bcf7187605) service to localhost/127.0.0.1:41675 2024-11-24T11:17:34,943 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/data/data1/current/BP-1825513930-172.17.0.3-1732446851790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T11:17:34,943 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/cluster_236b543a-2a7e-bfc3-6e3b-5c76750eb161/data/data2/current/BP-1825513930-172.17.0.3-1732446851790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T11:17:34,943 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T11:17:34,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T11:17:34,950 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T11:17:34,950 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T11:17:34,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T11:17:34,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5d77b4a4-6198-4ab1-d77e-c024c9c059ca/hadoop.log.dir/,STOPPED} 2024-11-24T11:17:34,964 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T11:17:35,085 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down